<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width" />
<title>All your tracing are belong to BPF | Trail of Bits Blog</title>
<link rel="profile" href="http://gmpg.org/xfn/11" />
<link rel="pingback" href="https://blog.trailofbits.com/xmlrpc.php" />
<!--[if lt IE 9]>
<script src="https://blog.trailofbits.com/wp-content/themes/everyday/js/html5.js" type="text/javascript"></script>
<![endif]-->

<meta name='robots' content='max-image-preview:large' />
<!-- Jetpack Site Verification Tags -->
<meta name="google-site-verification" content="Lbyyor8qrpJf-6qi1ql45JDKAmXUIcc3N0mFeVNwUok" />
<link rel='dns-prefetch' href='//js.hs-scripts.com' />
<link rel='dns-prefetch' href='//secure.gravatar.com' />
<link rel='dns-prefetch' href='//stats.wp.com' />
<link rel='dns-prefetch' href='//fonts-api.wp.com' />
<link rel='dns-prefetch' href='//widgets.wp.com' />
<link rel='dns-prefetch' href='//jetpack.wordpress.com' />
<link rel='dns-prefetch' href='//s0.wp.com' />
<link rel='dns-prefetch' href='//public-api.wordpress.com' />
<link rel='dns-prefetch' href='//0.gravatar.com' />
<link rel='dns-prefetch' href='//1.gravatar.com' />
<link rel='dns-prefetch' href='//2.gravatar.com' />
<link rel='dns-prefetch' href='//i0.wp.com' />
<link rel='dns-prefetch' href='//c0.wp.com' />
<link href='https://fonts.gstatic.com' crossorigin rel='preconnect' />
<link rel="alternate" type="application/rss+xml" title="Trail of Bits Blog &raquo; Feed" href="https://blog.trailofbits.com/feed/" />
<link rel="alternate" type="application/rss+xml" title="Trail of Bits Blog &raquo; Comments Feed" href="https://blog.trailofbits.com/comments/feed/" />
<link rel="alternate" type="application/rss+xml" title="Trail of Bits Blog &raquo; All your tracing are belong to BPF Comments Feed" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/feed/" />
<script type="text/javascript">
window._wpemojiSettings = {"baseUrl":"https:\/\/s.w.org\/images\/core\/emoji\/14.0.0\/72x72\/","ext":".png","svgUrl":"https:\/\/s.w.org\/images\/core\/emoji\/14.0.0\/svg\/","svgExt":".svg","source":{"concatemoji":"https:\/\/blog.trailofbits.com\/wp-includes\/js\/wp-emoji-release.min.js?ver=6.3"}};
/*! This file is auto-generated */
!function(i,n){var o,s,e;function c(e){try{var t={supportTests:e,timestamp:(new Date).valueOf()};sessionStorage.setItem(o,JSON.stringify(t))}catch(e){}}function p(e,t,n){e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(t,0,0);var t=new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data),r=(e.clearRect(0,0,e.canvas.width,e.canvas.height),e.fillText(n,0,0),new Uint32Array(e.getImageData(0,0,e.canvas.width,e.canvas.height).data));return t.every(function(e,t){return e===r[t]})}function u(e,t,n){switch(t){case"flag":return n(e,"\ud83c\udff3\ufe0f\u200d\u26a7\ufe0f","\ud83c\udff3\ufe0f\u200b\u26a7\ufe0f")?!1:!n(e,"\ud83c\uddfa\ud83c\uddf3","\ud83c\uddfa\u200b\ud83c\uddf3")&&!n(e,"\ud83c\udff4\udb40\udc67\udb40\udc62\udb40\udc65\udb40\udc6e\udb40\udc67\udb40\udc7f","\ud83c\udff4\u200b\udb40\udc67\u200b\udb40\udc62\u200b\udb40\udc65\u200b\udb40\udc6e\u200b\udb40\udc67\u200b\udb40\udc7f");case"emoji":return!n(e,"\ud83e\udef1\ud83c\udffb\u200d\ud83e\udef2\ud83c\udfff","\ud83e\udef1\ud83c\udffb\u200b\ud83e\udef2\ud83c\udfff")}return!1}function f(e,t,n){var r="undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?new OffscreenCanvas(300,150):i.createElement("canvas"),a=r.getContext("2d",{willReadFrequently:!0}),o=(a.textBaseline="top",a.font="600 32px Arial",{});return e.forEach(function(e){o[e]=t(a,e,n)}),o}function t(e){var t=i.createElement("script");t.src=e,t.defer=!0,i.head.appendChild(t)}"undefined"!=typeof Promise&&(o="wpEmojiSettingsSupports",s=["flag","emoji"],n.supports={everything:!0,everythingExceptFlag:!0},e=new Promise(function(e){i.addEventListener("DOMContentLoaded",e,{once:!0})}),new Promise(function(t){var n=function(){try{var e=JSON.parse(sessionStorage.getItem(o));if("object"==typeof e&&"number"==typeof e.timestamp&&(new Date).valueOf()<e.timestamp+604800&&"object"==typeof e.supportTests)return e.supportTests}catch(e){}return null}();if(!n){if("undefined"!=typeof Worker&&"undefined"!=typeof OffscreenCanvas&&"undefined"!=typeof URL&&URL.createObjectURL&&"undefined"!=typeof Blob)try{var e="postMessage("+f.toString()+"("+[JSON.stringify(s),u.toString(),p.toString()].join(",")+"));",r=new Blob([e],{type:"text/javascript"}),a=new Worker(URL.createObjectURL(r),{name:"wpTestEmojiSupports"});return void(a.onmessage=function(e){c(n=e.data),a.terminate(),t(n)})}catch(e){}c(n=f(s,u,p))}t(n)}).then(function(e){for(var t in e)n.supports[t]=e[t],n.supports.everything=n.supports.everything&&n.supports[t],"flag"!==t&&(n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&n.supports[t]);n.supports.everythingExceptFlag=n.supports.everythingExceptFlag&&!n.supports.flag,n.DOMReady=!1,n.readyCallback=function(){n.DOMReady=!0}}).then(function(){return e}).then(function(){var e;n.supports.everything||(n.readyCallback(),(e=n.source||{}).concatemoji?t(e.concatemoji):e.wpemoji&&e.twemoji&&(t(e.twemoji),t(e.wpemoji)))}))}((window,document),window._wpemojiSettings);
</script>
<style type="text/css">
img.wp-smiley,
img.emoji {
	display: inline !important;
	border: none !important;
	box-shadow: none !important;
	height: 1em !important;
	width: 1em !important;
	margin: 0 0.07em !important;
	vertical-align: -0.1em !important;
	background: none !important;
	padding: 0 !important;
}
</style>
	<link rel='stylesheet' id='all-css-ddcb23ffa248722bd643d383f3421230' href='https://blog.trailofbits.com/_static/??-eJyNUstyAiEQ/KHgxBxMckjlYlU+wA+wWJhClFcxg5v9+7DrbkrXqLkADT1Nz6NNQsXAGBiSK8YGgj1ykuoAPurikCCjk4xapEg8QwtF9NReS5hSYYPZQFOs09C4qA7C2SbL3AFx5/BWqJNdLCxMtvo+cbI57tsjBh0zyMLRS2arphdxtBpjykh0YQe15RowAIKBVFdspw9tUK7oWoA9ga9kiQ597+AcpOoXs3BopOoW3oaH0fXtHJ/HTCn6IqYs26Sipx1cZ8f4zScl0VptkMVQLoIqBuNNz/lvj+o/KYbKoPt1V3Gs2G+Ga0zVHQZlK/iKa6l1B5uTmZN2PzJ008l84Jw94A0TvKtFI8Aj5k7LB6M016WdzKgHd/3RBvMocltzBIrKSidcNJEuwF+tmyv0zRjPPfXTfyxX78vV89vq9eUHqO5WyA==' type='text/css' media='all' />
<style id='wp-block-library-inline-css'>
.has-text-align-justify{text-align:justify;}
</style>
<style id='classic-theme-styles-inline-css'>
/*! This file is auto-generated */
.wp-block-button__link{color:#fff;background-color:#32373c;border-radius:9999px;box-shadow:none;text-decoration:none;padding:calc(.667em + 2px) calc(1.333em + 2px);font-size:1.125em}.wp-block-file__button{background:#32373c;color:#fff;text-decoration:none}
</style>
<style id='global-styles-inline-css'>
body{--wp--preset--color--black: #000000;--wp--preset--color--cyan-bluish-gray: #abb8c3;--wp--preset--color--white: #ffffff;--wp--preset--color--pale-pink: #f78da7;--wp--preset--color--vivid-red: #cf2e2e;--wp--preset--color--luminous-vivid-orange: #ff6900;--wp--preset--color--luminous-vivid-amber: #fcb900;--wp--preset--color--light-green-cyan: #7bdcb5;--wp--preset--color--vivid-green-cyan: #00d084;--wp--preset--color--pale-cyan-blue: #8ed1fc;--wp--preset--color--vivid-cyan-blue: #0693e3;--wp--preset--color--vivid-purple: #9b51e0;--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple: linear-gradient(135deg,rgba(6,147,227,1) 0%,rgb(155,81,224) 100%);--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan: linear-gradient(135deg,rgb(122,220,180) 0%,rgb(0,208,130) 100%);--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange: linear-gradient(135deg,rgba(252,185,0,1) 0%,rgba(255,105,0,1) 100%);--wp--preset--gradient--luminous-vivid-orange-to-vivid-red: linear-gradient(135deg,rgba(255,105,0,1) 0%,rgb(207,46,46) 100%);--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray: linear-gradient(135deg,rgb(238,238,238) 0%,rgb(169,184,195) 100%);--wp--preset--gradient--cool-to-warm-spectrum: linear-gradient(135deg,rgb(74,234,220) 0%,rgb(151,120,209) 20%,rgb(207,42,186) 40%,rgb(238,44,130) 60%,rgb(251,105,98) 80%,rgb(254,248,76) 100%);--wp--preset--gradient--blush-light-purple: linear-gradient(135deg,rgb(255,206,236) 0%,rgb(152,150,240) 100%);--wp--preset--gradient--blush-bordeaux: linear-gradient(135deg,rgb(254,205,165) 0%,rgb(254,45,45) 50%,rgb(107,0,62) 100%);--wp--preset--gradient--luminous-dusk: linear-gradient(135deg,rgb(255,203,112) 0%,rgb(199,81,192) 50%,rgb(65,88,208) 100%);--wp--preset--gradient--pale-ocean: linear-gradient(135deg,rgb(255,245,203) 0%,rgb(182,227,212) 50%,rgb(51,167,181) 100%);--wp--preset--gradient--electric-grass: linear-gradient(135deg,rgb(202,248,128) 0%,rgb(113,206,126) 100%);--wp--preset--gradient--midnight: linear-gradient(135deg,rgb(2,3,129) 0%,rgb(40,116,252) 100%);--wp--preset--font-size--small: 13px;--wp--preset--font-size--medium: 20px;--wp--preset--font-size--large: 36px;--wp--preset--font-size--x-large: 42px;--wp--preset--font-family--albert-sans: 'Albert Sans';--wp--preset--font-family--alegreya: Alegreya;--wp--preset--font-family--arvo: Arvo;--wp--preset--font-family--bodoni-moda: 'Bodoni Moda';--wp--preset--font-family--cabin: Cabin;--wp--preset--font-family--chivo: Chivo;--wp--preset--font-family--commissioner: Commissioner;--wp--preset--font-family--cormorant: Cormorant;--wp--preset--font-family--courier-prime: 'Courier Prime';--wp--preset--font-family--crimson-pro: 'Crimson Pro';--wp--preset--font-family--dm-mono: 'DM Mono';--wp--preset--font-family--dm-sans: 'DM Sans';--wp--preset--font-family--domine: Domine;--wp--preset--font-family--eb-garamond: 'EB Garamond';--wp--preset--font-family--epilogue: Epilogue;--wp--preset--font-family--figtree: Figtree;--wp--preset--font-family--fira-sans: 'Fira Sans';--wp--preset--font-family--fraunces: Fraunces;--wp--preset--font-family--ibm-plex-mono: 'IBM Plex Mono';--wp--preset--font-family--ibm-plex-sans: 'IBM Plex Sans';--wp--preset--font-family--inter: Inter;--wp--preset--font-family--josefin-sans: 'Josefin Sans';--wp--preset--font-family--jost: Jost;--wp--preset--font-family--libre-baskerville: 'Libre Baskerville';--wp--preset--font-family--libre-franklin: 'Libre Franklin';--wp--preset--font-family--literata: Literata;--wp--preset--font-family--lora: Lora;--wp--preset--font-family--merriweather: Merriweather;--wp--preset--font-family--montserrat: Montserrat;--wp--preset--font-family--newsreader: Newsreader;--wp--preset--font-family--nunito: Nunito;--wp--preset--font-family--open-sans: 'Open Sans';--wp--preset--font-family--overpass: Overpass;--wp--preset--font-family--petrona: Petrona;--wp--preset--font-family--piazzolla: Piazzolla;--wp--preset--font-family--playfair-display: 'Playfair Display';--wp--preset--font-family--plus-jakarta-sans: 'Plus Jakarta Sans';--wp--preset--font-family--poppins: Poppins;--wp--preset--font-family--raleway: Raleway;--wp--preset--font-family--roboto-slab: 'Roboto Slab';--wp--preset--font-family--roboto: Roboto;--wp--preset--font-family--rubik: Rubik;--wp--preset--font-family--sora: Sora;--wp--preset--font-family--source-sans-pro: 'Source Sans Pro';--wp--preset--font-family--source-serif-pro: 'Source Serif Pro';--wp--preset--font-family--space-mono: 'Space Mono';--wp--preset--font-family--texturina: Texturina;--wp--preset--font-family--work-sans: 'Work Sans';--wp--preset--spacing--20: 0.44rem;--wp--preset--spacing--30: 0.67rem;--wp--preset--spacing--40: 1rem;--wp--preset--spacing--50: 1.5rem;--wp--preset--spacing--60: 2.25rem;--wp--preset--spacing--70: 3.38rem;--wp--preset--spacing--80: 5.06rem;--wp--preset--shadow--natural: 6px 6px 9px rgba(0, 0, 0, 0.2);--wp--preset--shadow--deep: 12px 12px 50px rgba(0, 0, 0, 0.4);--wp--preset--shadow--sharp: 6px 6px 0px rgba(0, 0, 0, 0.2);--wp--preset--shadow--outlined: 6px 6px 0px -3px rgba(255, 255, 255, 1), 6px 6px rgba(0, 0, 0, 1);--wp--preset--shadow--crisp: 6px 6px 0px rgba(0, 0, 0, 1);}:where(.is-layout-flex){gap: 0.5em;}:where(.is-layout-grid){gap: 0.5em;}body .is-layout-flow > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}body .is-layout-flow > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}body .is-layout-flow > .aligncenter{margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > .alignleft{float: left;margin-inline-start: 0;margin-inline-end: 2em;}body .is-layout-constrained > .alignright{float: right;margin-inline-start: 2em;margin-inline-end: 0;}body .is-layout-constrained > .aligncenter{margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > :where(:not(.alignleft):not(.alignright):not(.alignfull)){max-width: var(--wp--style--global--content-size);margin-left: auto !important;margin-right: auto !important;}body .is-layout-constrained > .alignwide{max-width: var(--wp--style--global--wide-size);}body .is-layout-flex{display: flex;}body .is-layout-flex{flex-wrap: wrap;align-items: center;}body .is-layout-flex > *{margin: 0;}body .is-layout-grid{display: grid;}body .is-layout-grid > *{margin: 0;}:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}.has-black-color{color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-color{color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-color{color: var(--wp--preset--color--white) !important;}.has-pale-pink-color{color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-color{color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-color{color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-color{color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-color{color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-color{color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-color{color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-color{color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-color{color: var(--wp--preset--color--vivid-purple) !important;}.has-black-background-color{background-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-background-color{background-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-background-color{background-color: var(--wp--preset--color--white) !important;}.has-pale-pink-background-color{background-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-background-color{background-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-background-color{background-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-background-color{background-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-background-color{background-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-background-color{background-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-background-color{background-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-background-color{background-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-background-color{background-color: var(--wp--preset--color--vivid-purple) !important;}.has-black-border-color{border-color: var(--wp--preset--color--black) !important;}.has-cyan-bluish-gray-border-color{border-color: var(--wp--preset--color--cyan-bluish-gray) !important;}.has-white-border-color{border-color: var(--wp--preset--color--white) !important;}.has-pale-pink-border-color{border-color: var(--wp--preset--color--pale-pink) !important;}.has-vivid-red-border-color{border-color: var(--wp--preset--color--vivid-red) !important;}.has-luminous-vivid-orange-border-color{border-color: var(--wp--preset--color--luminous-vivid-orange) !important;}.has-luminous-vivid-amber-border-color{border-color: var(--wp--preset--color--luminous-vivid-amber) !important;}.has-light-green-cyan-border-color{border-color: var(--wp--preset--color--light-green-cyan) !important;}.has-vivid-green-cyan-border-color{border-color: var(--wp--preset--color--vivid-green-cyan) !important;}.has-pale-cyan-blue-border-color{border-color: var(--wp--preset--color--pale-cyan-blue) !important;}.has-vivid-cyan-blue-border-color{border-color: var(--wp--preset--color--vivid-cyan-blue) !important;}.has-vivid-purple-border-color{border-color: var(--wp--preset--color--vivid-purple) !important;}.has-vivid-cyan-blue-to-vivid-purple-gradient-background{background: var(--wp--preset--gradient--vivid-cyan-blue-to-vivid-purple) !important;}.has-light-green-cyan-to-vivid-green-cyan-gradient-background{background: var(--wp--preset--gradient--light-green-cyan-to-vivid-green-cyan) !important;}.has-luminous-vivid-amber-to-luminous-vivid-orange-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-amber-to-luminous-vivid-orange) !important;}.has-luminous-vivid-orange-to-vivid-red-gradient-background{background: var(--wp--preset--gradient--luminous-vivid-orange-to-vivid-red) !important;}.has-very-light-gray-to-cyan-bluish-gray-gradient-background{background: var(--wp--preset--gradient--very-light-gray-to-cyan-bluish-gray) !important;}.has-cool-to-warm-spectrum-gradient-background{background: var(--wp--preset--gradient--cool-to-warm-spectrum) !important;}.has-blush-light-purple-gradient-background{background: var(--wp--preset--gradient--blush-light-purple) !important;}.has-blush-bordeaux-gradient-background{background: var(--wp--preset--gradient--blush-bordeaux) !important;}.has-luminous-dusk-gradient-background{background: var(--wp--preset--gradient--luminous-dusk) !important;}.has-pale-ocean-gradient-background{background: var(--wp--preset--gradient--pale-ocean) !important;}.has-electric-grass-gradient-background{background: var(--wp--preset--gradient--electric-grass) !important;}.has-midnight-gradient-background{background: var(--wp--preset--gradient--midnight) !important;}.has-small-font-size{font-size: var(--wp--preset--font-size--small) !important;}.has-medium-font-size{font-size: var(--wp--preset--font-size--medium) !important;}.has-large-font-size{font-size: var(--wp--preset--font-size--large) !important;}.has-x-large-font-size{font-size: var(--wp--preset--font-size--x-large) !important;}
:where(.wp-block-columns.is-layout-flex){gap: 2em;}:where(.wp-block-columns.is-layout-grid){gap: 2em;}
.wp-block-pullquote{font-size: 1.5em;line-height: 1.6;}
.wp-block-navigation a:where(:not(.wp-element-button)){color: inherit;}
:where(.wp-block-post-template.is-layout-flex){gap: 1.25em;}:where(.wp-block-post-template.is-layout-grid){gap: 1.25em;}
</style>
<link rel='stylesheet' id='dashicons-css' href='https://blog.trailofbits.com/wp-includes/css/dashicons.min.css?ver=6.3' media='all' />
<link rel='stylesheet' id='everyday-droid-serif-css' href='https://fonts-api.wp.com/css?family=Droid+Serif%3A400%2C700%2C400italic%2C700italic&#038;ver=20120328' media='all' />
<link rel='stylesheet' id='everyday-droid-sans-css' href='https://fonts-api.wp.com/css?family=Droid+Sans%3A400%2C700&#038;ver=20120328' media='all' />
<link rel='stylesheet' id='everyday-droid-mono-css' href='https://fonts-api.wp.com/css?family=Droid+Sans+Mono&#038;ver=20120328' media='all' />
<style id='jetpack-global-styles-frontend-style-inline-css'>
:root { --font-headings: unset; --font-base: unset; --font-headings-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif; --font-base-default: -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Oxygen-Sans,Ubuntu,Cantarell,"Helvetica Neue",sans-serif;}
</style>
<script id='jetpack_related-posts-js-extra'>
var related_posts_js_options = {"post_heading":"h4"};
</script>
<script type='text/javascript'  src='https://blog.trailofbits.com/wp-content/plugins/jetpack/_inc/build/related-posts/related-posts.min.js?m=1687202188'></script>
<script src='https://blog.trailofbits.com/wp-includes/js/jquery/jquery.min.js?ver=3.7.0' id='jquery-core-js'></script>
<script type='text/javascript'  src='https://blog.trailofbits.com/wp-includes/js/jquery/jquery-migrate.min.js?m=1686289764'></script>
<link rel="https://api.w.org/" href="https://blog.trailofbits.com/wp-json/" /><link rel="alternate" type="application/json" href="https://blog.trailofbits.com/wp-json/wp/v2/posts/101134" /><link rel="EditURI" type="application/rsd+xml" title="RSD" href="https://blog.trailofbits.com/xmlrpc.php?rsd" />

<link rel="canonical" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/" />
<link rel='shortlink' href='https://blog.trailofbits.com/?p=101134' />
<link rel="alternate" type="application/json+oembed" href="https://blog.trailofbits.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fblog.trailofbits.com%2F2021%2F11%2F09%2Fall-your-tracing-are-belong-to-bpf%2F" />
<link rel="alternate" type="text/xml+oembed" href="https://blog.trailofbits.com/wp-json/oembed/1.0/embed?url=https%3A%2F%2Fblog.trailofbits.com%2F2021%2F11%2F09%2Fall-your-tracing-are-belong-to-bpf%2F&#038;format=xml" />
			<!-- DO NOT COPY THIS SNIPPET! Start of Page Analytics Tracking for HubSpot WordPress plugin v10.2.1-->
			<script type="text/javascript" class="hsq-set-content-id" data-content-id="blog-post">
				var _hsq = _hsq || [];
				_hsq.push(["setContentType", "blog-post"]);
			</script>
			<!-- DO NOT COPY THIS SNIPPET! End of Page Analytics Tracking for HubSpot WordPress plugin -->
				<style>img#wpstats{display:none}</style>
		<style>
#page {
	background-color: #f7f7f7;
}
</style>

<meta name="description" content="By Alessandro Gario, Senior Software Engineer Originally published August 11, 2021 TL;DR: These simpler, step-by-step methods equip you to apply BPF tracing technology to real-word problems—no specialized tools or libraries required. BPF, a tracing technology in the Linux kernel for network stack tracing, has become popular recently thanks to new extensions that enable novel use-cases&hellip;" />
<style type="text/css" id="custom-background-css">
body.custom-background { background-image: url("https://blog.trailofbits.com/wp-content/themes/everyday/images/background.png"); background-position: left top; background-size: auto; background-repeat: repeat; background-attachment: scroll; }
</style>
				<style type="text/css">
				/* If html does not have either class, do not show lazy loaded images. */
				html:not( .jetpack-lazy-images-js-enabled ):not( .js ) .jetpack-lazy-image {
					display: none;
				}
			</style>
			<script>
				document.documentElement.classList.add(
					'jetpack-lazy-images-js-enabled'
				);
			</script>
		
<!-- Jetpack Open Graph Tags -->
<meta property="og:type" content="article" />
<meta property="og:title" content="All your tracing are belong to BPF" />
<meta property="og:url" content="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/" />
<meta property="og:description" content="By Alessandro Gario, Senior Software Engineer Originally published August 11, 2021 TL;DR: These simpler, step-by-step methods equip you to apply BPF tracing technology to real-word problems—no spec…" />
<meta property="article:published_time" content="2021-11-09T16:26:37+00:00" />
<meta property="article:modified_time" content="2021-11-09T16:26:37+00:00" />
<meta property="og:site_name" content="Trail of Bits Blog" />
<meta property="og:image" content="https://i0.wp.com/blog.trailofbits.com/wp-content/uploads/2020/07/cropped-android-chrome-512x512-1.png?fit=512%2C512&#038;ssl=1" />
<meta property="og:image:width" content="512" />
<meta property="og:image:height" content="512" />
<meta property="og:image:alt" content="" />
<meta property="og:locale" content="en_US" />
<meta name="twitter:creator" content="@trailofbits" />
<meta name="twitter:site" content="@trailofbits" />
<meta name="twitter:text:title" content="All your tracing are belong to BPF" />
<meta name="twitter:image" content="https://i0.wp.com/blog.trailofbits.com/wp-content/uploads/2020/07/cropped-android-chrome-512x512-1.png?fit=240%2C240&amp;ssl=1" />
<meta name="twitter:card" content="summary" />

<!-- End Jetpack Open Graph Tags -->
<link rel="icon" href="https://i0.wp.com/blog.trailofbits.com/wp-content/uploads/2020/07/cropped-android-chrome-512x512-1.png?fit=32%2C32&#038;ssl=1" sizes="32x32" />
<link rel="icon" href="https://i0.wp.com/blog.trailofbits.com/wp-content/uploads/2020/07/cropped-android-chrome-512x512-1.png?fit=192%2C192&#038;ssl=1" sizes="192x192" />
<link rel="apple-touch-icon" href="https://i0.wp.com/blog.trailofbits.com/wp-content/uploads/2020/07/cropped-android-chrome-512x512-1.png?fit=180%2C180&#038;ssl=1" />
<meta name="msapplication-TileImage" content="https://i0.wp.com/blog.trailofbits.com/wp-content/uploads/2020/07/cropped-android-chrome-512x512-1.png?fit=270%2C270&#038;ssl=1" />
<style type="text/css" id="wp-custom-css">/*
Welcome to Custom CSS!

To learn how this works, see http://wp.me/PEmnE-Bt
*/
.entry-content {
	padding-left: 0;
}

.entry-content
blockquote {
	margin-left: 25px;
}

.entry-content
.embed {
	margin-left: 0;
}

.entry-content
.embed-youtube {
	margin-left: 0;
}

/* To remove the infinite scroll footer HC-23308875-SK */
#infinite-footer
{
  display:none;
}

.wp-caption 
.wp-caption-text {
	text-align: center;
}

@media only screen and ( max-width: 600px ) {
    body {
        margin: 0;
        padding: 10px;
        overflow: auto;
        width: 100%
    }
}</style><!-- Your Google Analytics Plugin is missing the tracking ID -->
</head>

<body class="post-template-default single single-post postid-101134 single-format-standard custom-background group-blog">
<div id="page" class="hfeed site">
		<header id="masthead" class="site-header" role="banner">
		<hgroup>
			<h1 class="site-title"><a href="https://blog.trailofbits.com/" title="Trail of Bits Blog" rel="home">Trail of Bits Blog</a></h1>
			<h2 class="site-description"></h2>
					</hgroup>

		<nav role="navigation" class="site-navigation main-navigation">
			<h1 class="assistive-text">Menu</h1>
			<div class="assistive-text skip-link"><a href="#content" title="Skip to content">Skip to content</a></div>
			<div class="menu"><ul>
<li ><a href="https://blog.trailofbits.com/">Home</a></li></ul></div>
		</nav>
	</header><!-- #masthead .site-header -->

	<div id="main" class="main">
		<div id="primary" class="site-content">
			<div id="content" role="main">

			
				
<article id="post-101134" class="post-101134 post type-post status-publish format-standard hentry category-uncategorized entry">
	<header class="entry-header">

		<h1 class="entry-title">All your tracing are belong to BPF</h1>
					<ul class="entry-meta">
				<li class="entry-format">Post</li>
				<li class="permalink"><a href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/" title="11:26 am" rel="bookmark"><time class="entry-date" datetime="2021-11-09T11:26:37-05:00" pubdate>November 9, 2021</time></a></li>
				<li class="comment-link"><a href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/#respond">Leave a comment</a></li>
							</ul><!-- .entry-meta -->
			</header><!-- .entry-header -->

	<div class="entry-content">
		<p><strong>By Alessandro Gario, Senior Software Engineer</strong><br />
<em>Originally published August 11, 2021</em></p>
<p><strong>TL;DR:</strong> These simpler, step-by-step methods equip you to apply BPF tracing technology to real-word problems—no specialized tools or libraries required.</p>
<p>BPF, a tracing technology in the Linux kernel for network stack tracing, has become popular recently thanks to new extensions that enable novel use-cases outside of BPF’s original scope. Today it can be used to implement program performance analysis tools, system and program dynamic tracing utilities, and much more.</p>
<p>In this blog post we’ll show you how to use the Linux implementation of BPF to write tools that access system and program events. The <a href="https://github.com/iovisor/bcc/tree/master/tools">excellent tools from IO Visor</a> make it possible for users to easily harness BPF technology without the considerable time investment of writing specialized tools in native code languages.</p>
<h2>What the BPF?</h2>
<p>BPF itself is just a way to express a program, and a runtime interpreter for executing that program “safely.” It’s a set of specifications for virtual architecture, detailing how virtual machines dedicated to running its code should behave. The latest extensions to BPF have not only introduced new, really useful helper functions (such as reading a process’ memory), but also new registers and more stack space for the BPF bytecode.</p>
<p>Our main goal is to help you to take advantage of BPF and apply it to real-world problems without depending on external tools or libraries that may have been written with different goals and requirements in mind.</p>
<p>You can find the examples in this post <a href="https://github.com/trailofbits/bpf_tutorials_code">in our repository</a>. Please note that the code is simplified to focus on the concepts. This means that, where possible, we skip error checking and proper resource cleanup.</p>
<h2>BPF program limitations</h2>
<p>Even though we won’t be handwriting BPF assembly, it’s useful to know the code limitations since the in-kernel verifier will reject our instructions if we break its rules.</p>
<p>BPF programs are extremely simple, being made of only a single function. Instructions are sent to the kernel as an array of opcodes, meaning there’s no executable file format involved. Without sections, it’s not possible to have things like global variables or string literals; everything has to live on the stack, which can only hold up to 512 bytes. Branches are allowed, but it is only since kernel version 5.3 that jump opcodes can go backward—provided the verifier can prove the code will not execute forever.</p>
<p>The only other way to use loops without requiring recent kernel versions is to unroll them, but this will potentially use a lot of instructions, and older Linux versions will not load any program that exceeds the 4096 opcode count limit (see <strong>BPF_MAXINSNS</strong> under <strong>linux/bpf_common.h</strong>). Error handling in some cases is mandatory, and the verifier will prevent you from using resources that may fail initialization by rejecting the program.</p>
<p>These limitations are extremely important since these programs can get hooked on kernel code. When a verifier challenges the correctness of the code, it’s possible to prevent system crashes or slowdowns from loading malformed code.</p>
<h2>External resources</h2>
<p>To make BPF programs truly useful, they need ways to communicate with a user mode process and manage long-term data, i.e., via maps and perf event outputs.</p>
<p>Although many map types exist, they all essentially behave like key-value databases, and are commonly used to share data between user modes and/or other programs. Some of these types store data in per-CPU storage, making it easy to save and retrieve state when the same BPF program is run concurrently from different CPU cores.</p>
<p>Perf event outputs are generally used to send data to user mode programs and services, and are implemented as circular buffers.</p>
<h2>Event sources</h2>
<p>Without some data to process, our programs will just sit around doing nothing. BPF probes on Linux can be attached to several different event sources. For our purpose, we’re mainly interested in function tracing events.</p>
<h3>Dynamic instrumentation</h3>
<p>Similar to code hooking, BPF programs can be attached to any function. The probe type depends on where the target code lives. Kprobes are used when tracing kernel functions, while Uprobes are used when working with user mode libraries or binaries.</p>
<p>While Kprobe and Uprobe events are emitted when entering the monitored functions, Kretprobe and Uretprobe events are generated whenever the function returns. This works correctly even if the function being traced has multiple exit points. This kind of event does not forward typed syscall parameters and only comes with a <strong>pt_regs</strong> structure that contains the register values at the time of the call. Knowledge about the function prototype and system ABI is required to map back the function arguments to the right register.</p>
<h3>Static instrumentation</h3>
<p>It’s not always ideal to rely on function hooking when writing a tool, because the risk of breakage increases as the kernel or software gets updated. In most cases, it’s best to use a more stable event source such as a tracepoint.</p>
<p>There are two types of tracepoints:</p>
<ul>
<li>One for user mode code (USDT, a.k.a. User-Level Statically Defined Tracepoints)</li>
<li>One for kernel mode code (interestingly, they are referred to as just “tracepoints”).</li>
</ul>
<p>Both types of tracepoints are defined in the source code by the programmer, essentially defining a stable interface that shouldn’t change unless strictly necessary.</p>
<p>If DebugFS has been enabled and mounted, registered tracepoints will all appear under the /sys/kernel/debug/tracing folder. Similar to Kprobes and Kretprobes, each system call defined in the Linux kernel comes with two different tracepoints. The first one, <strong>sys_enter</strong>, is activated whenever a program in the system transitions to a syscall handler inside the kernel, and carries information about the parameters that have been received. The second (and last) one, <strong>sys_exit</strong>, only contains the exit code of the function and is invoked whenever the syscall function terminates.</p>
<h2>BPF development prerequisites</h2>
<p>Even though there’s no plan to use external libraries, we still have a few dependencies. The most important thing is to have access to a recent LLVM toolchain compiled with BPF support. If your system does not satisfy this requirement, it is possible—and actually encouraged—to make use of the <a href="https://github.com/osquery/osquery-toolchain">osquery toolchain</a>. You’ll also need CMake, as that’s what I use for the sample code.</p>
<p>When running inside the BPF environment, our programs make use of special helper functions that require a kernel version that’s at least above 4.18. While it’s possible to avoid using them, it would severely limit what we can do from our code.</p>
<p>Using Ubuntu 20.04 or equivalent is a good bet, as it comes with both a good kernel version and an up-to-date LLVM toolchain with BPF support.</p>
<p>Some LLVM knowledge is useful, but the code doesn’t require any advanced LLVM expertise. The <a href="https://llvm.org/docs/tutorial/MyFirstLanguageFrontend/index.html">Kaleidoscope</a> language tutorial on the official site is a great introduction if needed.</p>
<h2>Writing our first program</h2>
<p>There are many new concepts to introduce, so we’ll start simple: our first example loads a program that returns without doing anything.</p>
<p>First, we create a new LLVM module and a function that contains our logic:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
std::unique_ptr createBPFModule(llvm::LLVMContext &amp;context) {
  auto module = std::make_unique(&quot;BPFModule&quot;, context);
  module-&gt;setTargetTriple(&quot;bpf-pc-linux&quot;);
  module-&gt;setDataLayout(&quot;e-m:e-p:64:64-i64:64-n32:64-S128&quot;);
  
  return module;
}
  
std::unique_ptr generateBPFModule(llvm::LLVMContext &amp;context) {
  // Create the LLVM module for the BPF program
  auto module = createBPFModule(context);
  
  // BPF programs are made of a single function; we don't care about parameters
  // for the time being
  llvm::IRBuilder&lt;&gt; builder(context);
  auto function_type = llvm::FunctionType::get(builder.getInt64Ty(), {}, false);
  
  auto function = llvm::Function::Create(
      function_type, llvm::Function::ExternalLinkage, &quot;main&quot;, module.get());  
      
  // Ask LLVM to put this function in its own section, so we can later find it
  // more easily after we have compiled it to BPF code
  function-&gt;setSection(&quot;bpf_main_section&quot;);
  
  // Create the entry basic block and assemble the printk code using the helper
  // we have written
  auto entry_bb = llvm::BasicBlock::Create(context, &quot;entry&quot;, function);
  
  builder.SetInsertPoint(entry_bb);
  builder.CreateRet(builder.getInt64(0));
  
  return module;
}
</pre>
<p>Since we’re not going to handle event arguments, the function we created does not accept any parameters. Not much else is happening here except the return instruction. Remember, each BPF program has exactly one function, so it’s best to ask LLVM to store them in separate sections. This makes it easier to retrieve them once the module is compiled.</p>
<p>We can now JIT our module to BPF bytecode using the <strong>ExecutionEngine</strong> class from LLVM:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
SectionMap compileModule(std::unique_ptr module) {
  // Create a new execution engine builder and configure it
  auto exec_engine_builder =
      std::make_unique(std::move(module));
  
  exec_engine_builder-&gt;setMArch(&quot;bpf&quot;);
  
  SectionMap section_map;
  exec_engine_builder-&gt;setMCJITMemoryManager(
      std::make_unique(section_map));
      
  // Create the execution engine and build the given module
  std::unique_ptr execution_engine(
      exec_engine_builder-&gt;create());
      
  execution_engine-&gt;setProcessAllSections(true);
  execution_engine-&gt;finalizeObject();
  
  return section_map;
}
</pre>
<p>Our custom <strong>SectionMemoryManager</strong> class mostly acts as a passthrough to the original <strong>SectionMemoryManager</strong> class from LLVM—it’s only there to keep track of the sections that the <strong>ExecutionEngine</strong> object creates when compiling our IR.</p>
<p>Once the code is built, we get back a vector of bytes for each function that was created inside the module:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
int loadProgram(const std::vector &amp;program) {
  // The program needs to be aware how it is going to be used. We are
  // only interested in tracepoints, so we'll hardcode this value
  union bpf_attr attr = {};
  attr.prog_type = BPF_PROG_TYPE_TRACEPOINT;
  attr.log_level = 1U;
  
  // This is the array of (struct bpf_insn) instructions we have received
  // from the ExecutionEngine (see the compileModule() function for more
  // information)
  auto instruction_buffer_ptr = program.data();
  std::memcpy(&amp;attr.insns, &amp;instruction_buffer_ptr, sizeof(attr.insns));
  
  attr.insn_cnt =
      static_cast(program.size() / sizeof(struct bpf_insn));
      
  // The license is important because we will not be able to call certain
  // helpers within the BPF VM if it is not compatible
  static const std::string kProgramLicense{&quot;GPL&quot;};
  
  auto license_ptr = kProgramLicense.c_str();
  std::memcpy(&amp;attr.license, &amp;license_ptr, sizeof(attr.license));
  
  // The verifier will provide a text disasm of our BPF program in here.
  // If there is anything wrong with our code, we'll also find some
  // diagnostic output
  std::vector log_buffer(4096, 0);
  attr.log_size = static_cast&lt;__u32&gt;(log_buffer.size());
  
  auto log_buffer_ptr = log_buffer.data();
  std::memcpy(&amp;attr.log_buf, &amp;log_buffer_ptr, sizeof(attr.log_buf));
  
  auto program_fd =
      static_cast(::syscall(__NR_bpf, BPF_PROG_LOAD, &amp;attr, sizeof(attr)));
      
  if (program_fd &lt; 0) {
    std::cerr &lt;&lt; &quot;Failed to load the program: &quot; &lt;&lt; log_buffer.data() &lt;&lt; &quot;\n&quot;;
  }
  
  return program_fd;
}
</pre>
<p>Loading the program is not hard, but as you may have noticed, there is no helper function defined for the <strong>bpf()</strong> system call we’re using. The tracepoint is the easiest event type to set up, and it’s what we’re using for the time being.</p>
<p>Once the <strong>BPF_PROG_LOAD</strong> command is issued, the in-kernel verifier will validate our program and also provide a disassembly of it inside the log buffer we’ve provided. The operation will fail if kernel output is longer than the bytes available, so only provide a log buffer in production code if the load has already failed.</p>
<p>Another important field in the <strong>attr</strong> union is the program license; specifying any value other than GPL may disable some of the features that are exposed to BPF. I’m not a licensing expert, but it should be possible to use different licenses for the generator and the generated code (but please speak to a lawyer and/or your employer first!).</p>
<p>We can now assemble the <strong>main()</strong> function using the helpers we built:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
int main() {
  initializeLLVM();
  
  // Generate our BPF program
  llvm::LLVMContext context;
  auto module = generateBPFModule(context);
  
  // JIT the module to BPF code using the execution engine
  auto section_map = compileModule(std::move(module));
  if (section_map.size() != 1U) {
    std::cerr &lt;&lt; &quot;Unexpected section count\n&quot;;
    return 1;
  }
  
  // We have previously asked LLVM to create our function inside a specific
  // section; get our code back from it and load it
  const auto &amp;main_program = section_map.at(&quot;bpf_main_section&quot;);
  auto program_fd = loadProgram(main_program);
  if (program_fd &lt; 0) {
    return 1;
  }
  
  releaseLLVM();
  return 0;
}
</pre>
<p>If everything works correctly, no error is printed when the binary is run as the root user. You can find the source code for the empty program in the <strong>00-empty </strong>folder of the companion code repository.</p>
<p>But&#8230;this program isn’t very exciting, since it doesn’t do anything! Now we’ll update it so we can execute it when a certain system event happens.</p>
<h2>Creating our first useful program</h2>
<p>In order to actually execute our BPF programs, we have to attach them to an event source.</p>
<p>Creating a new tracepoint event is easy; it only involves reading and writing some files from under the <strong>debugfs</strong> folder:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
int createTracepointEvent(const std::string &amp;event_name) {
  const std::string kBaseEventPath = &quot;/sys/kernel/debug/tracing/events/&quot;;
  
  // This special file contains the id of the tracepoint, which is
  // required to initialize the event with perf_event_open  
  std::string event_id_path = kBaseEventPath + event_name + &quot;/id&quot;;
  
  // Read the tracepoint id and convert it to an integer
  auto event_file = std::fstream(event_id_path, std::ios::in);
  if (!event_file) {
    return -1;
  }
  
  std::stringstream buffer;
  buffer &lt;&lt; event_file.rdbuf();
  
  auto str_event_id = buffer.str();
  auto event_identifier = static_cast(
      std::strtol(str_event_id.c_str(), nullptr, 10));
      
  // Create the event
  struct perf_event_attr perf_attr = {};
  perf_attr.type = PERF_TYPE_TRACEPOINT;
  perf_attr.size = sizeof(struct perf_event_attr);
  perf_attr.config = event_identifier;
  perf_attr.sample_period = 1;
  perf_attr.sample_type = PERF_SAMPLE_RAW;
  perf_attr.wakeup_events = 1;
  perf_attr.disabled = 1;
  
  int process_id{-1};
  int cpu_index{0};
  
  auto event_fd =
      static_cast(::syscall(__NR_perf_event_open, &amp;perf_attr, process_id,
                                 cpu_index, -1, PERF_FLAG_FD_CLOEXEC));  
  
  return event_fd;
}
</pre>
<p>To create the event file descriptor, we have to find the tracepoint identifier, which is in a special file called (unsurprisingly) “id.”</p>
<p>For our last step, we attach the program to the tracepoint event we just created. This is trivial and can be done with a couple of <strong>ioctl</strong> calls on the event’s file descriptor:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
bool attachProgramToEvent(int event_fd, int program_fd) {
  if (ioctl(event_fd, PERF_EVENT_IOC_SET_BPF, program_fd) &lt; 0) {
    return false;
  }

  if (ioctl(event_fd, PERF_EVENT_IOC_ENABLE, 0) &lt; 0) {
    return false;
  }

  return true;
}
</pre>
<p>Our program should finally succeed in running our BPF code, but no output is generated yet since our module only really contained a return opcode. The easiest way to generate some output is to use the <strong>bpf_trace_printk</strong> helper to print a fixed string:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
void generatePrintk(llvm::IRBuilder&lt;&gt; &amp;builder) {
  // The bpf_trace_printk() function prototype can be found inside
  // the /usr/include/linux/bpf.h header file
  std::vector argument_type_list = {builder.getInt8PtrTy(),
                                                  builder.getInt32Ty()};

  auto function_type =
      llvm::FunctionType::get(builder.getInt64Ty(), argument_type_list, true);

  auto function =
      builder.CreateIntToPtr(builder.getInt64(BPF_FUNC_trace_printk),
                             llvm::PointerType::getUnqual(function_type));

  // Allocate 8 bytes on the stack
  auto buffer = builder.CreateAlloca(builder.getInt64Ty());

  // Copy the string characters to the 64-bit integer
  static const std::string kMessage{&quot;Hello!!&quot;};

  std::uint64_t message{0U};
  std::memcpy(&amp;message, kMessage.c_str(), sizeof(message));

  // Store the characters inside the buffer we allocated on the stack
  builder.CreateStore(builder.getInt64(message), buffer);

  // Print the characters
  auto buffer_ptr = builder.CreateBitCast(buffer, builder.getInt8PtrTy());

#if LLVM_VERSION_MAJOR &lt; 11
  auto function_callee = function;
#else
  auto function_callee = llvm::FunctionCallee(function_type, function);
#endif

  builder.CreateCall(function_callee, {buffer_ptr, builder.getInt32(8U)});
}
</pre>
<p>Importing new helper functions from BPF is quite easy. The first thing we need is the prototype, which can be taken from the <strong>linux/bpf.h</strong> include header. The one relative to <strong>printk</strong> reads as follows:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
 * int bpf_trace_printk(const char *fmt, u32 fmt_size, ...)
 * 	Description
 * 		This helper is a &quot;printk()-like&quot; facility for debugging. It
 * 		prints a message defined by format *fmt* (of size *fmt_size*)
 * 		to file *\/sys/kernel/debug/tracing/trace* from DebugFS, if
 * 		available. It can take up to three additional **u64**
 * 		arguments (as an eBPF helpers, the total number of arguments is
 * 		limited to five).
</pre>
<p>Once the function type matches, we only have to assemble a call that uses the helper function ID as the destination address: <strong>BPF_FUNC_trace_printk</strong>. The <strong>generatePrintk</strong> function can now be added to our program right before we create the return instruction inside <strong>generateBPFModule</strong>.</p>
<p>The full source code for this program can be found in the <strong>01-hello_open</strong> folder.</p>
<p>Running the program again will show the “Hello!!” string inside the <strong>/sys/kernel/debug/tracing/trace_pipe</strong> file every time the tracepoint event is emitted. Using text output can be useful, but due to the BPF VM limitations the printf helper is not as useful as can be in a standard C program.</p>
<p>In the next section, we’ll take a look at maps and how to use them as data storage for our programs.</p>
<h2>Profiling system calls</h2>
<h3>Using maps to store data</h3>
<p>Maps are a major component in most programs, and can be used in a number of different ways. Since they’re accessible from both kernel and user mode, they can be useful in storing data for later processing either from additional probes or user programs. Given the limitations that BPF imposes, they’re also commonly used to provide scratch space for handling temporary data that does not fit on the stack.</p>
<p>There are many map types; some are specialized for certain uses, such as storing stack traces. Others are more generic, and suitable for use as custom data containers.</p>
<p>Concurrency and thread safety are not just user mode problems, and BPF comes with two really useful special map types that have dedicated storage for storing values in CPU scope. These maps are commonly used to replace the stack, as a per-CPU map can be easily referenced by programs without having to worry about synchronization.</p>
<p>It’s rather simple to create and use maps since they all share the same interface, regardless of type. The following table, taken from the BPF header file comments, documents the most common operations:</p>
<li>‍<strong>BPF_MAP_CREATE:</strong> Create a map and return a file descriptor that refers to the map. The close-on-exec file descriptor flag (see <strong>fcntl(2))</strong> is automatically enabled for the new file descriptor.</li>
<li>‍<strong>BPF_MAP_LOOKUP_ELEM:</strong> Look up an element by key in a specified map and return its value.</li>
<li>‍<strong>BPF_MAP_UPDATE_ELEM:</strong> Create or update an element (key/value pair) in a specified map.</li>
<li>‍<strong>BPF_MAP_DELETE_ELEM:</strong> Look up and delete an element by key in a specified map.
<p>The only important thing to remember is that when operating on per-CPU maps the value is not just a single entry, but an array of values that has as many items as CPU cores.</p>
<h3>Creating a map</h3>
<p>Before we can create our map, we have to determine which type we want to use. The following enum declaration has been taken from the linux/bpf.h header file:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
enum bpf_map_type {
  BPF_MAP_TYPE_UNSPEC,  /* Reserve 0 as invalid map type */
  BPF_MAP_TYPE_HASH,
  BPF_MAP_TYPE_ARRAY,
  BPF_MAP_TYPE_PROG_ARRAY,
  BPF_MAP_TYPE_PERF_EVENT_ARRAY,
  BPF_MAP_TYPE_PERCPU_HASH,
  BPF_MAP_TYPE_PERCPU_ARRAY,
  BPF_MAP_TYPE_STACK_TRACE,
  BPF_MAP_TYPE_CGROUP_ARRAY,
  BPF_MAP_TYPE_LRU_HASH,
  BPF_MAP_TYPE_LRU_PERCPU_HASH,
  BPF_MAP_TYPE_LPM_TRIE,
  BPF_MAP_TYPE_ARRAY_OF_MAPS,
  BPF_MAP_TYPE_HASH_OF_MAPS,
  BPF_MAP_TYPE_DEVMAP,
  BPF_MAP_TYPE_SOCKMAP,
  BPF_MAP_TYPE_CPUMAP,
};
</pre>
<p>Most of the time we’ll use hash maps and arrays. We have to create a <strong>bpf_attr</strong> union, initializing key and value size as well as the maximum amount of entries it can hold.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
int createMap(bpf_map_type type, std::uint32_t key_size,
              std::uint32_t value_size, std::uint32_t key_count) {

  union bpf_attr attr = {};

  attr.map_type = type;
  attr.key_size = key_size;
  attr.value_size = value_size;
  attr.max_entries = key_count;

  return static_cast(
      syscall(__NR_bpf, BPF_MAP_CREATE, &amp;attr, sizeof(attr)));
}
</pre>
<p>Not every available operation always makes sense for all map types. For example, it’s not possible to delete entries when working with an array. Lookup operations are also going to behave differently, as they will only fail when the specified index is beyond the last element.</p>
<p>Here’s the code to read a value from a map:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Error codes for map operations; depending on the map type, reads may
// return NotFound if the specified key is not present
enum class ReadMapError { Succeeded, NotFound, Failed };

// Attempts to read a key from the specified map. Values in per-CPU maps
// actually have multiple entries (one per CPU)
ReadMapError readMapKey(std::vector &amp;value, int map_fd,
                        const void *key) {

  union bpf_attr attr = {};

  // Use memcpy to avoid string aliasing issues
  attr.map_fd = static_cast&lt;__u32&gt;(map_fd);
  std::memcpy(&amp;attr.key, &amp;key, sizeof(attr.key));

  auto value_ptr = value.data();
  std::memcpy(&amp;attr.value, &amp;value_ptr, sizeof(attr.value));

  auto err =
      ::syscall(__NR_bpf, BPF_MAP_LOOKUP_ELEM, &amp;attr, sizeof(union bpf_attr));

  if (err &gt;= 0) {
    return ReadMapError::Succeeded;
  }

  if (errno == ENOENT) {
    return ReadMapError::NotFound;
  } else {
    return ReadMapError::Failed;
  }
}
</pre>
<h2>Writing a BPF program to count syscall invocations</h2>
<p>In this example we’ll build a probe that counts how many times the tracepoint we&#8217;re tracing gets called. We’ll create a counter for each processor core, using a per-CPU array map that only contains a single item.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
auto map_fd = createMap(BPF_MAP_TYPE_PERCPU_ARRAY, 4U, 8U, 1U);
if (map_fd &lt; 0) {
  return 1;
}
</pre>
<p>Referencing this map from the BPF code is not too hard but requires some additional operations:</p>
<ol>
<li>Convert the map file descriptor to a map address</li>
<li>Use the bpf_map_lookup_elem helper function to retrieve the pointer to the desired map entry</li>
<li>Check the returned pointer to make sure the operation has succeeded (the validator will reject our program otherwise)</li>
<li>Update the counter value</li>
</ol>
<p>The map address can be obtained through a special LLVM intrinsic called “<strong>pseudo</strong>.”</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Returns the pseudo intrinsic, useful to convert file descriptors (like maps
// and perf event outputs) to map addresses so they can be used from the BPF VM
llvm::Function *getPseudoFunction(llvm::IRBuilder&lt;&gt; &amp;builder) {
  auto &amp;insert_block = *builder.GetInsertBlock();
  auto &amp;module = *insert_block.getModule();

  auto pseudo_function = module.getFunction(&quot;llvm.bpf.pseudo&quot;);

  if (pseudo_function == nullptr) {
    // clang-format off
    auto pseudo_function_type = llvm::FunctionType::get(
      builder.getInt64Ty(),

      {
        builder.getInt64Ty(),
        builder.getInt64Ty()
      },

      false
    );
    // clang-format on

    pseudo_function = llvm::Function::Create(pseudo_function_type,
                                             llvm::GlobalValue::ExternalLinkage,
                                             &quot;llvm.bpf.pseudo&quot;, module);
  }

  return pseudo_function;
}

// Converts the given (map or perf event output) file descriptor to a map
// address
llvm::Value *mapAddressFromFileDescriptor(int fd, llvm::IRBuilder&lt;&gt; &amp;builder) {
  auto pseudo_function = getPseudoFunction(builder);

  // clang-format off
  auto map_integer_address_value = builder.CreateCall(
    pseudo_function,

    {
      builder.getInt64(BPF_PSEUDO_MAP_FD),
      builder.getInt64(static_cast(fd))
    }
  );
  // clang-format on

  return builder.CreateIntToPtr(map_integer_address_value,
                                builder.getInt8PtrTy());
}
</pre>
<p>Importing the <strong>bpf_map_lookup_elem</strong> helper function follows the same procedure we used to import the <strong>bpf_trace_printk</strong> one. Looking at the <strong>linux/bpf.h</strong>, the prototype reads:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
 * void *bpf_map_lookup_elem(struct bpf_map *map, const void *key)
 * 	Description
 * 		Perform a lookup in *map* for an entry associated to *key*.
 * 	Return
 * 		Map value associated to *key*, or **NULL** if no entry was
 * 		found.
</pre>
<p>Notice how the key parameter is passed by pointer and not by value. We’ll have to allocate the actual key on the stack using <strong>CreateAlloca</strong>. Since allocations should always happen in the first (entry) basic block, our function will accept a pre-filled buffer as key. The return type is a void pointer, but we can save work if we directly declare the function with the correct value type.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Attempts to retrieve a pointer to the specified key inside the map_fd map
llvm::Value *bpfMapLookupElem(llvm::IRBuilder&lt;&gt; &amp;builder, llvm::Value *key,
                              llvm::Type *value_type, int map_fd) {

  std::vector argument_type_list = {builder.getInt8PtrTy(),
                                                  builder.getInt32Ty()};

  auto function_type = llvm::FunctionType::get(value_type-&gt;getPointerTo(),
                                               argument_type_list, false);

  auto function =
      builder.CreateIntToPtr(builder.getInt64(BPF_FUNC_map_lookup_elem),
                             llvm::PointerType::getUnqual(function_type));

  auto map_address = mapAddressFromFileDescriptor(map_fd, builder);

#if LLVM_VERSION_MAJOR &lt; 11
  auto function_callee = function;
#else
  auto function_callee = llvm::FunctionCallee(function_type, function);
#endif

  return builder.CreateCall(function_callee, {map_address, key});
}
</pre>
<p>Back to the BPF program generator, we can now call the new <strong>bpfMapLookupElem</strong> to retrieve the first value in our array map:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
auto map_key_buffer = builder.CreateAlloca(builder.getInt32Ty());
builder.CreateStore(builder.getInt32(0U), map_key_buffer);

auto counter_ptr =
    bpfMapLookupElem(builder, map_key_buffer, builder.getInt32Ty(), map_fd);
</pre>
<p>Since we are using a per-CPU array map, the pointer that returns from this function references a private array entry for the core we’re running on. Before we can use it, however, we have to test whether the function has succeeded; otherwise, the verifier will reject the program. This is trivial and can be done with a comparison instruction and a new basic block.</p>
<pre class="brush: plain; title: ; notranslate" title="">
auto null_ptr = llvm::Constant::getNullValue(counter_ptr-&gt;getType());
auto cond = builder.CreateICmpEQ(null_ptr, counter_ptr);

auto error_bb = llvm::BasicBlock::Create(context, &quot;error&quot;, function);
auto continue_bb = llvm::BasicBlock::Create(context, &quot;continue&quot;, function);

builder.CreateCondBr(cond, error_bb, continue_bb);

builder.SetInsertPoint(error_bb);
builder.CreateRet(builder.getInt64(0));

builder.SetInsertPoint(continue_bb);
</pre>
<p>The pointer to the counter value can now be dereferenced without causing a validation error from the verifier.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
auto counter = builder.CreateLoad(counter_ptr);
auto new_value = builder.CreateAdd(counter, builder.getInt32(1));

builder.CreateStore(new_value, counter_ptr);
builder.CreateRet(builder.getInt64(0));
</pre>
<p>There is no need to import and use the <strong>bpf_map_update_elem()</strong> helper function since we can directly increment the value from the pointer we received. We only have to load the value from the pointer, increment it, and then store it back where it was.</p>
<p>Once we have finished with our tracer, we can retrieve the counters and inspect them:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
auto processor_count = getProcessorCount();
std::vector value(processor_count * sizeof(std::uint64_t));

std::uint32_t key{0U};
auto map_error = readMapKey(value, map_fd, &amp;key);

if (map_error != ReadMapError::Succeeded) {
  std::cerr &lt;&lt; &quot;Failed to read from the map\n&quot;;
  return 1;
}

std::vector per_cpu_counters(processor_count);
std::memcpy(per_cpu_counters.data(), value.data(), value.size());
</pre>
<p>When dealing with per-CPU maps, it is important to not rely on <strong>get_nprocs_conf</strong> and use <strong>/sys/devices/system/cpu/possible </strong>instead. On VMware Fusion for example, the vcpu.hotadd setting will cause Linux to report 128 possible CPUs when enabled, regardless of how many cores have been actually assigned to the virtual machine.</p>
<p>The full sample code can be found in the <strong>02-syscall_counter</strong> folder.</p>
<p>One interesting experiment is to attach this program to the system call tracepoint used by the <strong>chmod</strong> command line tool to update file modes. The <strong>strace</strong> debugging utility can help determine which syscall is being used. In this case we are going to be monitoring the following tracepoint: <strong>syscalls/sys_enter_fchmodat</strong>.</p>
<p>The <strong>taskset</strong> command can be altered to force the <strong>fchmodat</strong> syscall to be called from a specific processor:</p>
<pre>
taskset 1 chmod /path/to/file # CPU 1
taskset 2 chmod /path/to/file # CPU 2
</pre>
<h2>Using perf event outputs</h2>
<p>Maps can be a really powerful way to store data for later processing, but it’s impossible for user mode programs to know when and where new data is available for reading.</p>
<p>Perf event outputs can help solve this problem, since they enable the program to be notified whenever new data is available. Additionally, since they behave like a circular buffer, we do not have the same size limitations we have when setting map values.</p>
<p>In this section, we’ll build an application that can measure how much time it takes to handle a system call. To make this work, we’ll attach a program to both the entry and exit points of a tracepoint to gather timestamps.</p>
<h3>Initialization</h3>
<p>Before we start creating our perf output, we have to create a structure to hold our resources. In total, we’ll have a file descriptor for the map and then a perf output per processor, along with its own memory mapping.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
struct PerfEventArray final {
  int fd;

  std::vector output_fd_list;
  std::vector mapped_memory_pointers;
};
</pre>
<p>To initialize it, we have to create a BPF map of the type <strong>PERF_EVENT_ARRAY</strong> first. This special data structure maps a specific CPU index to a private perf event output specified as a file descriptor. For it to function properly, we must use the following parameters when creating the map:</p>
<ol>
<li>Key size must be set to 4 bytes (CPU index).</li>
<li>Value size must be set to 4 bytes (size of a file descriptor specified with an <strong>int</strong>).</li>
<li>Entry count must be set to a value greater than or equal to the number of processors.</li>
</ol>
<pre class="brush: cpp; title: ; notranslate" title="">
auto processor_count = getProcessorCount();

// Create the perf event array map
obj.fd = createMap(BPF_MAP_TYPE_PERF_EVENT_ARRAY, 4U, 4U, processor_count);
if (obj.fd &lt; 0) {
  return false;
}
</pre>
<p>When we looked at maps in the previous sections, we only focused on reading. For the next steps we also need to write new values, so let’s take a look at how to set keys.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
ReadMapError setMapKey(std::vector &amp;value, int map_fd,
                       const void *key) {

  union bpf_attr attr = {};
  attr.flags = BPF_ANY; // Always set the value
  attr.map_fd = static_cast&lt;__u32&gt;(map_fd);

  // Use memcpy to avoid string aliasing issues
  std::memcpy(&amp;attr.key, &amp;key, sizeof(attr.key));

  auto value_ptr = value.data();
  std::memcpy(&amp;attr.value, &amp;value_ptr, sizeof(attr.value));

  auto err = ::syscall(__NR_bpf, BPF_MAP_UPDATE_ELEM, &amp;attr, sizeof(attr));

  if (err &lt; 0) {
    return ReadMapError::Failed;
  }

  return ReadMapError::Succeeded;
}
</pre>
<p>This is not too different from how we read map values, but this time we don’t have to deal with the chance that the key may not be present. As always when dealing with per-CPU maps, the data pointer should be considered as an array containing one value per CPU.</p>
<p>The next step is to create a perf event output for each online processor with the <strong>perf_event_open</strong> system call, using the special <strong>PERF_COUNT_SW_BPF_OUTPUT</strong> config value.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
struct perf_event_attr attr {};
attr.type = PERF_TYPE_SOFTWARE;
attr.size = sizeof(attr);
attr.config = PERF_COUNT_SW_BPF_OUTPUT;
attr.sample_period = 1;
attr.sample_type = PERF_SAMPLE_RAW;
attr.wakeup_events = 1;

std::uint32_t processor_index;
for (processor_index = 0U; processor_index &lt; processor_count;
     ++processor_index) {

  // clang-format off
  auto perf_event_fd = ::syscall(
    __NR_perf_event_open,
    &amp;attr,
    -1,               // Process ID (unused)
    processor_index,  // 0 -&gt; getProcessorCount()
    -1,               // Group ID (unused)
    0                 // Flags (unused)
  );
  // clang-format on

  if (perf_event_fd == -1) {
    return false;
  }

  obj.output_fd_list.push_back(static_cast(perf_event_fd));
}
</pre>
<p>Now that we have the file descriptors, we can populate the perf event array map we created:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Set the perf event output file descriptors inside the map
processor_index = 0U;

for (auto perf_event_fd : obj.output_fd_list) {
  std::vector value(4);
  std::memcpy(value.data(), &amp;perf_event_fd, sizeof(perf_event_fd));

  auto err = setMapKey(value, obj.fd, &amp;processor_index);
  if (err != ReadMapError::Succeeded) {
    return false;
  }

  ++processor_index;
}
</pre>
<p>Finally, we create a memory mapping for each perf output:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Create a memory mapping for each output
auto size = static_cast(1 + std::pow(2, page_count));
size *= static_cast(getpagesize());

for (auto &amp;perf_event_fd : obj.output_fd_list) {
  auto ptr = mmap(nullptr,                // Desired base address (unused)
                  size,                   // Mapped memory size
                  PROT_READ | PROT_WRITE, // Memory protection
                  MAP_SHARED,             // Flags
                  perf_event_fd,          // The perf output handle
                  0                       // Offset (unused)
  );

  if (ptr == MAP_FAILED) {
    return false;
  }

  obj.mapped_memory_pointers.push_back(ptr);
}
</pre>
<p>This is the memory we’ll read from when capturing the BPF program output.</p>
<h3>Writing a BPF program to profile system calls</h3>
<p>Now that we have a file descriptor of the perf event array map, we can use it from within the BPF code to send data with the <strong>bpf_perf_event_output</strong> helper function. Here’s the prototype from <strong>linux/bpf.h</strong>:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
 * int bpf_perf_event_output(struct pt_reg *ctx, struct bpf_map *map, u64 flags, void *data, u64 size)
 * 	Description
 * 		Write raw *data* blob into a special BPF perf event held by
 * 		*map* of type **BPF_MAP_TYPE_PERF_EVENT_ARRAY**. This perf
 * 		event must have the following attributes: **PERF_SAMPLE_RAW**
 * 		as **sample_type**, **PERF_TYPE_SOFTWARE** as **type**, and
 * 		**PERF_COUNT_SW_BPF_OUTPUT** as **config**.
 *
 * 		The *flags* are used to indicate the index in *map* for which
 * 		the value must be put, masked with **BPF_F_INDEX_MASK**.
 * 		Alternatively, *flags* can be set to **BPF_F_CURRENT_CPU**
 * 		to indicate that the index of the current CPU core should be
 * 		used.
 *
 * 		The value to write, of *size*, is passed through eBPF stack and
 * 		pointed by *data*.
 *
 * 		The context of the program *ctx* needs also be passed to the
 * 		helper.
</pre>
<p>The <strong>ctx</strong> parameter must be always set to the value of the first argument received in the entry point function of the BPF program.</p>
<p>The map address is obtained with the LLVM pseudo intrinsic that we imported in the previous section. Data and size are self-explanatory, but it is important to remember that the memory pointer must reside inside the BPF program (i.e., we can’t pass a user pointer).</p>
<p>The last parameter, flags, can be used as a CPU index mask to select the perf event output this data should be sent to. A special value can be passed to ask the BPF VM to automatically use the index of the processor we’re running on.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Sends the specified buffer to the map_fd perf event output
llvm::Value *bpfPerfEventOutput(llvm::IRBuilder&lt;&gt; &amp;builder, llvm::Value *ctx,
                                int map_fd, std::uint64_t flags,
                                llvm::Value *data, llvm::Value *size) {

  // clang-format off
  std::vector argument_type_list = {
    // Context
    ctx-&gt;getType(),

    // Map address
    builder.getInt8PtrTy(),

    // Flags
    builder.getInt64Ty(),

    // Data pointer
    data-&gt;getType(),

    // Size
    builder.getInt64Ty()
  };
  // clang-format on

  auto function_type =
      llvm::FunctionType::get(builder.getInt32Ty(), argument_type_list, false);

  auto function =
      builder.CreateIntToPtr(builder.getInt64(BPF_FUNC_perf_event_output),
                             llvm::PointerType::getUnqual(function_type));

  auto map_address = mapAddressFromFileDescriptor(map_fd, builder);

#if LLVM_VERSION_MAJOR &lt; 11
  auto function_callee = function;
#else
  auto function_callee = llvm::FunctionCallee(function_type, function);
#endif

  return builder.CreateCall(
      function_callee, {ctx, map_address, builder.getInt64(flags), data, size});
}
</pre>
<p>The file descriptor and flags parameters are most likely known at compile time, so we can make the function a little more user friendly by accepting integer types. The buffer size, however, is often determined at runtime, so it’s best to use an <strong>llvm::Value</strong> pointer.</p>
<p>While it’s possible to just send the raw timestamps whenever we enter and leave the system call of our choice, it’s much easier and more efficient to compute what we need directly inside the BPF code. To do this we’ll use a per-CPU hash map shared across two different BPF programs: one for the sys_enter event, and another one for the sys_exit.</p>
<p>From the enter program, we’ll save the system timestamp in the map. When the exit program is invoked, we’ll retrieve it and use it to determine how much time it took. The resulting value is then sent to the user mode program using the perf output.</p>
<p>Creating the map is easy, and we can re-use the map helpers we wrote in the previous sections. Both the timestamp and the map key are 64-bit values, so we’ll use 8 bytes for both:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
auto map_fd = createMap(BPF_MAP_TYPE_HASH, 8U, 8U, 100U);
if (map_fd &lt; 0) {
  std::cerr &lt;&lt; &quot;Failed to create the map\n&quot;;
  return 1;
}
</pre>
<h3>Writing the enter program</h3>
<p>We will need to generate a key for our map. A combination of the process ID and thread ID is a good candidate for this:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
 * u64 bpf_get_current_pid_tgid(void)
 * 	Return
 * 		A 64-bit integer containing the current tgid and pid, and
 * 		created as such:
 * 		*current_task*\ **-&gt;tgid &lt;&lt; 32 \|**
 * 		*current_task*\ **-&gt;pid**.
</pre>
<p>Then the system timestamp needs to be acquired. Even though the <strong>ktime_get_ns</strong> helper function counts the time from the boot, it’s still a good alternative since we only have to use it to calculate the execution time.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
 * u64 bpf_ktime_get_ns(void)
 *  Description
 *    Return the time elapsed since system boot, in nanoseconds.
 *  Return
 *    Current *ktime*.
</pre>
<p>By now you should be well versed in importing them, so here are the two definitions:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Returns a 64-bit integer that contains both the process and thread id
llvm::Value *bpfGetCurrentPidTgid(llvm::IRBuilder&lt;&gt; &amp;builder) {
  auto function_type = llvm::FunctionType::get(builder.getInt64Ty(), {}, false);

  auto function =
      builder.CreateIntToPtr(builder.getInt64(BPF_FUNC_get_current_pid_tgid),
                             llvm::PointerType::getUnqual(function_type));

#if LLVM_VERSION_MAJOR &lt; 11
  auto function_callee = function;
#else
  auto function_callee = llvm::FunctionCallee(function_type, function);
#endif

  return builder.CreateCall(function_callee, {});
}

// Returns the amount of nanoseconds elapsed from system boot
llvm::Value *bpfKtimeGetNs(llvm::IRBuilder&lt;&gt; &amp;builder) {
  auto function_type = llvm::FunctionType::get(builder.getInt64Ty(), {}, false);

  auto function =
      builder.CreateIntToPtr(builder.getInt64(BPF_FUNC_ktime_get_ns),
                             llvm::PointerType::getUnqual(function_type));

#if LLVM_VERSION_MAJOR &lt; 11
  auto function_callee = function;
#else
  auto function_callee = llvm::FunctionCallee(function_type, function);
#endif

  return builder.CreateCall(function_callee, {});
}
</pre>
<p>We can now use the newly defined functions to generate a map key and acquire the system timestamp:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Map keys and values are passed by pointer; create two buffers on the
// stack and initialize them
auto map_key_buffer = builder.CreateAlloca(builder.getInt64Ty());
auto timestamp_buffer = builder.CreateAlloca(builder.getInt64Ty());

auto current_pid_tgid = bpfGetCurrentPidTgid(builder);
builder.CreateStore(current_pid_tgid, map_key_buffer);

auto timestamp = bpfKtimeGetNs(builder);
builder.CreateStore(timestamp, timestamp_buffer);
</pre>
<p>For this program we have replaced the array map we used in the previous sections with a hash map. It’s no longer possible to use the <strong>bpf_map_lookup_elem()</strong> helper since the map key we have will fail with <strong>ENOENT</strong> if the element does not exist.</p>
<p>To fix this, we have to import a new helper named <strong>bpf_map_update_elem()</strong>:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
* int bpf_map_update_elem(struct bpf_map *map, const void *key, const void *value, u64 flags)
* 	Description
* 		Add or update the value of the entry associated to *key* in
* 		*map* with *value*. *flags* is one of:
*
* 		**BPF_NOEXIST**
* 			The entry for *key* must not exist in the map.
* 		**BPF_EXIST**
* 			The entry for *key* must already exist in the map.
* 		**BPF_ANY**
* 			No condition on the existence of the entry for *key*.
*
* 		Flag value **BPF_NOEXIST** cannot be used for maps of types
* 		**BPF_MAP_TYPE_ARRAY** or **BPF_MAP_TYPE_PERCPU_ARRAY**  (all
* 		elements always exist), the helper would return an error.
* 	Return
* 		0 on success, or a negative error in case of failure.
</pre>
<p>We’ll keep the map file descriptor and flag values as integers, since we know their values before the module is compiled.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Updates the value of the specified key inside the map_fd BPF map
llvm::Value *bpfMapUpdateElem(llvm::IRBuilder&lt;&gt; &amp;builder, int map_fd,
                              llvm::Value *key, llvm::Value *value,
                              std::uint64_t flags) {

  // clang-format off
  std::vector argument_type_list = {
    // Map address
    builder.getInt8PtrTy(),

    // Key
    key-&gt;getType(),

    // Value
    value-&gt;getType(),

    // Flags
    builder.getInt64Ty()
  };
  // clang-format on

  auto function_type =
      llvm::FunctionType::get(builder.getInt64Ty(), argument_type_list, false);

  auto function =
      builder.CreateIntToPtr(builder.getInt64(BPF_FUNC_map_update_elem),
                             llvm::PointerType::getUnqual(function_type));

  auto map_address = mapAddressFromFileDescriptor(map_fd, builder);

#if LLVM_VERSION_MAJOR &lt; 11
  auto function_callee = function;
#else
  auto function_callee = llvm::FunctionCallee(function_type, function);
#endif

  return builder.CreateCall(function_callee,
                            {map_address, key, value, builder.getInt64(flags)});
}
</pre>
<p>We can now store the timestamp inside the map and close the enter program:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Save the timestamp inside the map
bpfMapUpdateElem(builder, map_fd, map_key_buffer, timestamp_buffer, BPF_ANY);
builder.CreateRet(builder.getInt64(0));
</pre>
<h3>‍Writing the exit program</h3>
<p>In this program, we’ll retrieve the timestamp we stored and use it to measure how much time we’ve spent inside the system call. Once we have the result, we’ll send it to user mode using the perf output.</p>
<p>When creating the <strong>llvm::Function</strong> for this program, we must define at least one argument. This value will be required later for the <strong>ctx</strong> parameter that we have to pass to the <strong>bpf_perf_event_output()</strong> helper.</p>
<p>First, we have to acquire the map entry; as always, we must check for any possible error or the verifier will not let us load our program.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Create the entry basic block
auto entry_bb = llvm::BasicBlock::Create(context, &quot;entry&quot;, function);
builder.SetInsertPoint(entry_bb);

// Map keys are passed by pointer; create a buffer on the stack and initialize
// it
auto map_key_buffer = builder.CreateAlloca(builder.getInt64Ty());
auto current_pid_tgid = bpfGetCurrentPidTgid(builder);
builder.CreateStore(current_pid_tgid, map_key_buffer);

// Check the pointer and make sure the lookup has succeeded; this is
// mandatory, or the BPF verifier will refuse to load our program
auto timestamp_ptr =
    bpfMapLookupElem(builder, map_key_buffer, builder.getInt64Ty(), map_fd);

auto null_ptr = llvm::Constant::getNullValue(timestamp_ptr-&gt;getType());
auto cond = builder.CreateICmpEQ(null_ptr, timestamp_ptr);

auto error_bb = llvm::BasicBlock::Create(context, &quot;error&quot;, function);
auto continue_bb = llvm::BasicBlock::Create(context, &quot;continue&quot;, function);

builder.CreateCondBr(cond, error_bb, continue_bb);

// Terminate the program if the pointer is not valid
builder.SetInsertPoint(error_bb);
builder.CreateRet(builder.getInt64(0));

// In this new basic block, the pointer is valid
builder.SetInsertPoint(continue_bb);
</pre>
<p>Next, we want to read our previous timestamp and subtract it from the current time:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Read back the old timestamp and obtain the current one
auto enter_timestamp = builder.CreateLoad(timestamp_ptr);
auto exit_timestamp = bpfKtimeGetNs(builder);

// Measure how much it took to go from the first instruction to the return
auto time_consumed = builder.CreateSub(exit_timestamp, enter_timestamp);
</pre>
<p>The <strong>bpf_perf_event_output</strong> expects a buffer, so we have to store our result somewhere in memory. We can re-use the map value address so we don’t have to allocate more stack space:</p>
<pre class="brush: cpp; title: ; notranslate" title="">builder.CreateStore(time_consumed, timestamp_ptr);
</pre>
<p>Remember, we have to pass the first program argument to the <strong>ctx</strong> parameter; the <strong>arg_begin</strong> method of an <strong>llvm::Function </strong>will return exactly that. When sending data, the <strong>bpf_perf_event_output()</strong> helper expects a pointer. We can re-use the timestamp pointer we obtained from the map and avoid allocating additional memory to the very limited stack we have:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
builder.CreateStore(time_consumed, timestamp_ptr);

// Send the result to the perf event array
auto ctx = function-&gt;arg_begin();
bpfPerfEventOutput(builder, ctx, perf_fd, static_cast(-1UL),
                   timestamp_ptr, builder.getInt64(8U));
</pre>
<p>Using <strong>-1UL</strong> as the flag value means that BPF will automatically send this data to the perf event output associated with the CPU we’re running on.</p>
<h3>Reading data from the perf outputs</h3>
<p>In our user mode program, we can access the perf buffers through the memory mappings we created. The list of perf event output descriptors can be used together with the <strong>poll()</strong> function using an array of <strong>pollfd</strong> structures. When one of the <strong>fd</strong> we have set is readable, the corresponding memory mapping will contain the data sent by the BPF program.</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Uses poll() to wait for the next event happening on the perf even toutput
bool waitForPerfData(std::vector &amp;readable_outputs,
                     const PerfEventArray &amp;obj, int timeout) {

  readable_outputs = {};

  // Collect all the perf event output file descriptors inside a
  // pollfd structure
  std::vector poll_fd_list;
  for (auto fd : obj.output_fd_list) {
    struct pollfd poll_fd = {};
    poll_fd.fd = fd;
    poll_fd.events = POLLIN;

    poll_fd_list.push_back(std::move(poll_fd));
  }

  // Use poll() to determine which outputs are readable
  auto err = ::poll(poll_fd_list.data(), poll_fd_list.size(), timeout);
  if (err &lt; 0) {
    if (errno == EINTR) {
      return true;
    }

    return false;

  } else if (err == 0) {
    return true;
  }

  // Save the index of the outputs that can be read inside the vector
  for (auto it = poll_fd_list.begin(); it != poll_fd_list.end(); ++it) {
    auto ready = ((it-&gt;events &amp; POLLIN) != 0);

    if (ready) {
      auto index = static_cast(it - poll_fd_list.begin());
      readable_outputs.push_back(index);
    }
  }

  return true;
}
</pre>
<p>Inside the memory we have mapped, the <a href="https://github.com/torvalds/linux/blob/3f59dbcace56fae7e4ed303bab90f1bedadcfdf4/include/uapi/linux/perf_event.h#L478">perf_event_mmap_page header</a> will describe the properties and boundaries of the allocated circular buffer.</p>
<p>The structure is too big to be reported here, but the most important fields are:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
__u64  data_head;   /* head in the data section */
__u64  data_tail;   /* user-space written tail */
__u64  data_offset; /* where the buffer starts */
__u64  data_size;   /* data buffer size */
</pre>
<p>The base of the data allocation is located at the offset <strong>data_offset</strong>; to find the start of our buffer, however, we have to add it to the <strong>data_tail</strong> value, making sure to wrap around whenever we exceed the data allocation size specified by the <strong>data_size</strong> field:</p>
<pre>buffer_start = mapped_memory + data_offset + (data_tail % data_size)</pre>
<p>Similarly, the data_head field can be used to find the end of the buffer:</p>
<pre>buffer_end = mapped_memory + data_offset + (data_head % data_size)</pre>
<p>If the end of the buffer is at a lower offset compared to the start, then data is wrapping at the <strong>data_size</strong> edge and the read has to happen with two operations.</p>
<p>When extracting data, the program is expected to confirm the read by updating the <strong>data_tail</strong> value and adding the number of bytes processed, while the kernel will advance the <strong>data_head</strong> field automatically as new bytes are received. Data is lost when the <strong>data_head</strong> offset wraps around and crosses <strong>data_tail</strong>; a special structure inside this buffer will warn the program if this happens.</p>
<p>Program data is packaged inside the data we have just extracted, preceded by two headers. The first one is the <strong>perf_event_header</strong> structure:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
struct perf_event_header {
  u32 type;
  u16 misc;
  u16 size;
};
</pre>
<p>The second one is an additional 32-bit size field that accounts for itself and the data that follows. Multiple consecutive writes from the BPF program may be added under the same object. Data is, however, grouped by type, which can be used to determine what kind of data to expect after the header. When using BPF, we’ll only have to deal with either our data or a notification of type <strong>PERF_RECORD_LOST</strong>, which is used to inform the program that a <strong>bpf_perf_event_output()</strong> call has overwritten data in the ring buffer before we could have a chance to read it.</p>
<p>Here’s some annotated code that shows how the whole procedure works:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
using PerfBuffer = std::vector;
using PerfBufferList = std::vector;

// Reads from the specified perf event array, appending new bytes to the
// perf_buffer_context. When a new complete buffer is found, it is moved
// inside the the 'data' vector
bool readPerfEventArray(PerfBufferList &amp;data,
                        PerfBufferList &amp;perf_buffer_context,
                        const PerfEventArray &amp;obj, int timeout) {

  // Keep track of the offsets we are interested in to avoid
  // strict aliasing issues
  static const auto kDataOffsetPos{
      offsetof(struct perf_event_mmap_page, data_offset)};

  static const auto kDataSizePos{
      offsetof(struct perf_event_mmap_page, data_size)};

  static const auto kDataTailPos{
      offsetof(struct perf_event_mmap_page, data_tail)};

  static const auto kDataHeadPos{
      offsetof(struct perf_event_mmap_page, data_head)};

  data = {};

  if (perf_buffer_context.empty()) {
    auto processor_count = getProcessorCount();
    perf_buffer_context.resize(processor_count);
  }

  // Use poll() to determine which perf event outputs are readable
  std::vector readable_outputs;
  if (!waitForPerfData(readable_outputs, obj, timeout)) {
    return false;
  }

  for (auto perf_output_index : readable_outputs) {
    // Read the static header fields
    auto perf_memory = static_cast(
        obj.mapped_memory_pointers.at(perf_output_index));

    std::uint64_t data_offset{};
    std::memcpy(&amp;data_offset, perf_memory + kDataOffsetPos, 8U);

    std::uint64_t data_size{};
    std::memcpy(&amp;data_size, perf_memory + kDataSizePos, 8U);

    auto edge = perf_memory + data_offset + data_size;

    for (;;) {
      // Read the dynamic header fields
      std::uint64_t data_head{};
      std::memcpy(&amp;data_head, perf_memory + kDataHeadPos, 8U);

      std::uint64_t data_tail{};
      std::memcpy(&amp;data_tail, perf_memory + kDataTailPos, 8U);

      if (data_head == data_tail) {
        break;
      }

      // Determine where the buffer starts and where it ends, taking into
      // account the fact that it may wrap around
      auto start = perf_memory + data_offset + (data_tail % data_size);
      auto end = perf_memory + data_offset + (data_head % data_size);

      auto byte_count = data_head - data_tail;
      auto read_buffer = PerfBuffer(byte_count);

      if (end &lt; start) {
        auto bytes_until_wrap = static_cast(edge - start);
        std::memcpy(read_buffer.data(), start, bytes_until_wrap);

        auto remaining_bytes =
            static_cast(end - (perf_memory + data_offset));

        std::memcpy(read_buffer.data() + bytes_until_wrap,
                    perf_memory + data_offset, remaining_bytes);

      } else {
        std::memcpy(read_buffer.data(), start, byte_count);
      }

      // Append the new data to our perf buffer
      auto &amp;perf_buffer = perf_buffer_context[perf_output_index];

      auto insert_point = perf_buffer.size();
      perf_buffer.resize(insert_point + read_buffer.size());

      std::memcpy(&amp;perf_buffer[insert_point], read_buffer.data(),
                  read_buffer.size());

      // Confirm the read
      std::memcpy(perf_memory + kDataTailPos, &amp;data_head, 8U);
    }
  }

  // Extract the data from the buffers we have collected
  for (auto &amp;perf_buffer : perf_buffer_context) {
    // Get the base header
    struct perf_event_header header = {};
    if (perf_buffer.size() &lt; sizeof(header)) {
      continue;
    }

    std::memcpy(&amp;header, perf_buffer.data(), sizeof(header));
    if (header.size &gt; perf_buffer.size()) {
      continue;
    }

    if (header.type == PERF_RECORD_LOST) {
      std::cout &lt;&lt; &quot;One or more records have been lost\n&quot;;

    } else {
      // Determine the buffer boundaries
      auto buffer_ptr = perf_buffer.data() + sizeof(header);
      auto buffer_end = perf_buffer.data() + header.size;

      for (;;) {
        if (buffer_ptr + 4U &gt;= buffer_end) {
          break;
        }

        // Note: this is data_size itself + bytes used for the data
        std::uint32_t data_size = {};
        std::memcpy(&amp;data_size, buffer_ptr, 4U);

        buffer_ptr += 4U;
        data_size -= 4U;

        if (buffer_ptr + data_size &gt;= buffer_end) {
          break;
        }

        auto program_data = PerfBuffer(data_size);
        std::memcpy(program_data.data(), buffer_ptr, data_size);
        data.push_back(std::move(program_data));

        buffer_ptr += 8U;
        data_size -= 8U;
      }
    }

    // Erase the chunk we consumed from the buffer
    perf_buffer.erase(perf_buffer.begin(), perf_buffer.begin() + header.size);
  }

  return true;
}
</pre>
<h3>Writing the main function</h3>
<p>While it is entirely possible (and sometimes useful, in order to share types) to use a single LLVM module and context for both the enter and exit programs, we will create two different modules to avoid changing the previous sample code we’ve built.</p>
<p>The program generation goes through the usual steps, but now we are loading two instead of one, so the previous code has been changed to reflect that.</p>
<p>The new and interesting part is the main loop where the perf event output data is read and processed:</p>
<pre class="brush: cpp; title: ; notranslate" title="">
// Incoming data is appended here
PerfBufferList perf_buffer;

std::uint64_t total_time_used{};
std::uint64_t sample_count{};

std::cout &lt;&lt; &quot;Tracing average time used to service the following syscall: &quot;
          &lt;&lt; kSyscallName &lt;&lt; &quot;\n&quot;;

std::cout &lt;&lt; &quot;Collecting samples for 10 seconds...\n&quot;;

auto start_time = std::chrono::system_clock::now();

for (;;) {
  // Data that is ready for processing is moved inside here
  PerfBufferList data;
  if (!readPerfEventArray(data, perf_buffer, perf_event_array, 1)) {
    std::cerr &lt;&lt; &quot;Failed to read from the perf event array\n&quot;;
    return 1;
  }

  // Inspect the buffers we have received
  for (const auto &amp;buffer : data) {
    if (buffer.size() != 8U) {
      std::cout &lt;&lt; &quot;Unexpected buffer size: &quot; &lt;&lt; buffer.size() &lt;&lt; &quot;\n&quot;;
      continue;
    }

    // Read each sample and update the counters; use memcpy to avoid
    // strict aliasing issues
    std::uint64_t time_used{};
    std::memcpy(&amp;time_used, buffer.data(), 8U);

    total_time_used += time_used;
    ++sample_count;

    std::cout &lt;&lt; time_used &lt;&lt; &quot;ns\n&quot;;
  }

  // Exit after 10 seconds
  auto elapsed_msecs = std::chrono::duration_cast(
                           std::chrono::system_clock::now() - start_time)
                           .count();

  if (elapsed_msecs &gt; 10000) {
    break;
  }
}

// Print a summary of the data we have collected
std::cout &lt;&lt; &quot;Total time used: &quot; &lt;&lt; total_time_used &lt;&lt; &quot; nsecs\n&quot;;
std::cout &lt;&lt; &quot;Sample count: &quot; &lt;&lt; sample_count &lt;&lt; &quot;\n&quot;;
std::cout &lt;&lt; &quot;Average: &quot; &lt;&lt; (total_time_used / sample_count) &lt;&lt; &quot; nsecs\n&quot;;
</pre>
<p>The full source code can be found in the <strong>03-syscall_profiler</strong> folder.</p>
<p>Running the sample program as root should print something similar to the following output:</p>
<pre>
Tracing average time used to service the following syscall: fchmodat
Collecting samples for 10 seconds...
178676ns
72886ns
80481ns
147897ns
171152ns
80803ns
69208ns
75273ns
76981ns
Total time used: 953357 nsecs
Sample count: 9
Average: 105928 nsecs
</pre>
<h2>Writing a BPF program to do ANYTHING</h2>
<p>BPF is in active development and is becoming more and more useful with each update, enabling new use cases that extend the original vision. Recently, newly added BPF functionality allowed us to write a simple <a href="https://github.com/trailofbits/ebpfault">system-wide syscall fault injector</a> using nothing but BPF and a compatible kernel that supported the required <strong>bpf_override_return</strong> functionality.</p>
<p>If you want to keep up with how this technology evolves, one of the best places to start with is <a href="https://www.brendangregg.com/blog/index.html">Brendan’s Gregg blog</a>. The <a href="https://github.com/iovisor">IO Visor Project repository</a> also contains a ton of code and documentation that is extremely useful if you plan on writing your own BPF-powered tools.</p>
<p>Want to integrate BPF into your products? We can help! <a href="https://www.trailofbits.com/contact/">Contact us today</a>, and <a href="https://github.com/trailofbits/ebpfpub">check out our ebpfpub library</a>.</p>
<div class="sharedaddy sd-sharing-enabled"><div class="robots-nocontent sd-block sd-social sd-social-icon-text sd-sharing"><h3 class="sd-title">Share this:</h3><div class="sd-content"><ul><li class="share-twitter"><a rel="nofollow noopener noreferrer" data-shared="sharing-twitter-101134" class="share-twitter sd-button share-icon" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/?share=twitter" target="_blank" title="Click to share on Twitter" ><span>Twitter</span></a></li><li class="share-linkedin"><a rel="nofollow noopener noreferrer" data-shared="sharing-linkedin-101134" class="share-linkedin sd-button share-icon" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/?share=linkedin" target="_blank" title="Click to share on LinkedIn" ><span>LinkedIn</span></a></li><li class="share-reddit"><a rel="nofollow noopener noreferrer" data-shared="" class="share-reddit sd-button share-icon" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/?share=reddit" target="_blank" title="Click to share on Reddit" ><span>Reddit</span></a></li><li class="share-telegram"><a rel="nofollow noopener noreferrer" data-shared="" class="share-telegram sd-button share-icon" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/?share=telegram" target="_blank" title="Click to share on Telegram" ><span>Telegram</span></a></li><li class="share-facebook"><a rel="nofollow noopener noreferrer" data-shared="sharing-facebook-101134" class="share-facebook sd-button share-icon" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/?share=facebook" target="_blank" title="Click to share on Facebook" ><span>Facebook</span></a></li><li class="share-pocket"><a rel="nofollow noopener noreferrer" data-shared="" class="share-pocket sd-button share-icon" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/?share=pocket" target="_blank" title="Click to share on Pocket" ><span>Pocket</span></a></li><li class="share-email"><a rel="nofollow noopener noreferrer" data-shared="" class="share-email sd-button share-icon" href="mailto:?subject=%5BShared%20Post%5D%20All%20your%20tracing%20are%20belong%20to%20BPF&body=https%3A%2F%2Fblog.trailofbits.com%2F2021%2F11%2F09%2Fall-your-tracing-are-belong-to-bpf%2F&share=email" target="_blank" title="Click to email a link to a friend" data-email-share-error-title="Do you have email set up?" data-email-share-error-text="If you&#039;re having problems sharing via email, you might not have email set up for your browser. You may need to create a new email yourself." data-email-share-nonce="bf67e090ec" data-email-share-track-url="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/?share=email"><span>Email</span></a></li><li class="share-print"><a rel="nofollow noopener noreferrer" data-shared="" class="share-print sd-button share-icon" href="https://blog.trailofbits.com/2021/11/09/all-your-tracing-are-belong-to-bpf/#print" target="_blank" title="Click to print" ><span>Print</span></a></li><li class="share-end"></li></ul></div></div></div><div class='sharedaddy sd-block sd-like jetpack-likes-widget-wrapper jetpack-likes-widget-unloaded' id='like-post-wrapper-3681601-101134-64df9cca89f84' data-src='https://widgets.wp.com/likes/#blog_id=3681601&amp;post_id=101134&amp;origin=blog.trailofbits.com&amp;obj_id=3681601-101134-64df9cca89f84' data-name='like-post-frame-3681601-101134-64df9cca89f84' data-title='Like or Reblog'><h3 class="sd-title">Like this:</h3><div class='likes-widget-placeholder post-likes-widget-placeholder' style='height: 55px;'><span class='button'><span>Like</span></span> <span class="loading">Loading...</span></div><span class='sd-text-color'></span><a class='sd-link-color'></a></div>
<div id='jp-relatedposts' class='jp-relatedposts' >
	<h3 class="jp-relatedposts-headline"><em>Related</em></h3>
</div>			</div><!-- .entry-content -->

	<footer class="entry-footer">
		<p class="entry-byline">By Trent Brunson</p>
		<p class="cat-links">Posted in <a href="https://blog.trailofbits.com/category/uncategorized/" rel="category tag">Uncategorized</a></p>
			</footer>
</article><!-- #post-101134 -->
						<nav role="navigation" id="nav-below" class="site-navigation post-navigation">
			<h1 class="assistive-text">Post navigation</h1>

		
			<div class="nav-previous"><a href="https://blog.trailofbits.com/2021/11/09/privacyraven-implementing-a-proof-of-concept-for-model-inversion/" rel="prev"><span class="meta-nav">&larr;</span> PrivacyRaven: Implementing a proof of concept for model inversion</a></div>			<div class="nav-next"><a href="https://blog.trailofbits.com/2021/11/10/announcing-osquery-5-now-with-endpointsecurity-on-macos/" rel="next">Announcing osquery 5: Now with EndpointSecurity on macOS <span class="meta-nav">&rarr;</span></a></div>
		
		</nav><!-- #nav-below -->
		
					<div id="comments" class="comments-area">
	
	
	
	
	
		<div id="respond" class="comment-respond">
			<h3 id="reply-title" class="comment-reply-title">Leave a Reply<small><a rel="nofollow" id="cancel-comment-reply-link" href="/2021/11/09/all-your-tracing-are-belong-to-bpf/#respond" style="display:none;">Cancel reply</a></small></h3>			<form id="commentform" class="comment-form">
				<iframe
					title="Comment Form"
					src="https://jetpack.wordpress.com/jetpack-comment/?blogid=3681601&#038;postid=101134&#038;comment_registration=0&#038;require_name_email=1&#038;stc_enabled=1&#038;stb_enabled=1&#038;show_avatars=1&#038;avatar_default=identicon&#038;greeting=Leave+a+Reply&#038;jetpack_comments_nonce=b564f5ea75&#038;greeting_reply=Leave+a+Reply+to+%25s&#038;color_scheme=light&#038;lang=en_US&#038;jetpack_version=12.5-a.11&#038;show_cookie_consent=10&#038;has_cookie_consent=0&#038;token_key=%3Bnormal%3B&#038;sig=86744884fa3c81bef8a63e24dc7ccc3167d7cf14#parent=https%3A%2F%2Fblog.trailofbits.com%2F2021%2F11%2F09%2Fall-your-tracing-are-belong-to-bpf%2F"
											name="jetpack_remote_comment"
						style="width:100%; height: 430px; border:0;"
										class="jetpack_remote_comment"
					id="jetpack_remote_comment"
					sandbox="allow-same-origin allow-top-navigation allow-scripts allow-forms allow-popups"
				>
									</iframe>
									<!--[if !IE]><!-->
					<script>
						document.addEventListener('DOMContentLoaded', function () {
							var commentForms = document.getElementsByClassName('jetpack_remote_comment');
							for (var i = 0; i < commentForms.length; i++) {
								commentForms[i].allowTransparency = false;
								commentForms[i].scrolling = 'no';
							}
						});
					</script>
					<!--<![endif]-->
							</form>
		</div>

		
		<input type="hidden" name="comment_parent" id="comment_parent" value="" />

		
</div><!-- #comments .comments-area -->

			
			</div><!-- #content -->
		</div><!-- #primary .site-content -->

		<div id="secondary" class="widget-area" role="complementary">
						<aside id="search-4" class="widget widget_search">	<form method="get" id="searchform" action="https://blog.trailofbits.com/" role="search">
		<label for="s" class="assistive-text">Search</label>
		<input type="search" class="field" name="s" id="s" placeholder="Search &hellip;" />
		<input type="submit" class="submit" name="submit" id="searchsubmit" value="Search" />
	</form>
</aside><aside id="text-112416548" class="widget widget_text"><h1 class="widget-title">About Us</h1>			<div class="textwidget"><p>Since 2012, Trail of Bits has helped secure some of the world’s most targeted organizations and products. We combine high-­end security research with a real­ world attacker mentality to reduce risk and fortify code.</p>
<p>Read more at <a href="https://www.trailofbits.com">www.trailofbits.com</a></p>
</div>
		</aside><aside id="rss_links-4" class="widget widget_rss_links"><h1 class="widget-title">Subscribe via RSS</h1><p><a target="_self" href="https://blog.trailofbits.com/feed/" title="Subscribe to posts"><img src="https://blog.trailofbits.com/wp-content/plugins/jetpack/images/rss/red-small.png" alt="RSS feed" />&nbsp;RSS - Posts</a></p>
</aside>
		<aside id="recent-posts-5" class="widget widget_recent_entries">
		<h1 class="widget-title">Recent Posts</h1>
		<ul>
											<li>
					<a href="https://blog.trailofbits.com/2023/08/14/can-you-pass-the-rekt-test/">Can you pass the Rekt test?</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/08/09/use-our-suite-of-ebpf-libraries/">Use our suite of eBPF libraries</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/08/02/a-mistake-in-the-bulletproofs-paper-could-have-led-to-the-theft-of-millions-of-dollars/">A mistake in the bulletproofs paper could have led to the theft of millions of dollars</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/31/how-ai-will-affect-cybersecurity-what-we-told-the-cftc/">How AI will affect cybersecurity: What we told the CFTC</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/28/the-future-of-clang-based-tooling/">The future of Clang-based tooling</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/26/announcing-the-trail-of-bits-testing-handbook/">Announcing the Trail of Bits Testing Handbook</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/21/fuzzing-on-chain-contracts-with-echidna/">Fuzzing on-chain contracts with Echidna</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/18/trail-of-bitss-response-to-ostp-national-priorities-for-ai-rfi/">Trail of Bits’s Response to OSTP National Priorities for AI RFI</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/14/evaluating-blockchain-security-maturity/">Evaluating blockchain security maturity</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/12/what-we-told-the-cftc-about-crypto-threats/">What we told the CFTC about blockchain threats</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/07/07/differential-fuzz-testing-upgradeable-smart-contracts-with-diffusc/">Differential fuzz testing upgradeable smart contracts with Diffusc</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/06/16/trail-of-bitss-response-to-ntia-ai-accountability-rfc/">Trail of Bits’s Response to NTIA AI Accountability RFC</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/06/15/finding-bugs-with-mlir-and-vast/">Finding bugs in C code with Multi-Level IR and VAST</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/05/23/trusted-publishing-a-new-benchmark-for-packaging-security/">Trusted publishing: a new benchmark for packaging security</a>
									</li>
											<li>
					<a href="https://blog.trailofbits.com/2023/05/16/real-world-crypto-2023-recap/">Real World Crypto 2023 Recap</a>
									</li>
					</ul>

		</aside><aside id="text-112416546" class="widget widget_text"><h1 class="widget-title">Yearly Archive</h1>			<div class="textwidget"><ul>
<li><a href="https://blog.trailofbits.com/2023">2023</a></li>
<li><a href="https://blog.trailofbits.com/2022">2022</a></li>
<li><a href="https://blog.trailofbits.com/2021">2021</a></li>
<li><a href="https://blog.trailofbits.com/2020">2020</a></li>
<li><a href="https://blog.trailofbits.com/2019">2019</a></li>
<li><a href="https://blog.trailofbits.com/2018">2018</a></li>
<li><a href="https://blog.trailofbits.com/2017">2017</a></li>
<li><a href="https://blog.trailofbits.com/2016">2016</a></li>
<li><a href="https://blog.trailofbits.com/2015">2015</a></li>
<li><a href="https://blog.trailofbits.com/2014">2014</a></li>
<li><a href="https://blog.trailofbits.com/2013">2013</a></li>
<li><a href="https://blog.trailofbits.com/2012">2012</a></li>
</ul></div>
		</aside><aside id="categories-4" class="widget widget_categories"><h1 class="widget-title">Categories</h1>
			<ul>
					<li class="cat-item cat-item-291"><a href="https://blog.trailofbits.com/category/apple/">Apple</a> (13)
</li>
	<li class="cat-item cat-item-141319"><a href="https://blog.trailofbits.com/category/attacks/">Attacks</a> (11)
</li>
	<li class="cat-item cat-item-685590231"><a href="https://blog.trailofbits.com/category/audits/">Audits</a> (7)
</li>
	<li class="cat-item cat-item-44620"><a href="https://blog.trailofbits.com/category/authentication/">Authentication</a> (5)
</li>
	<li class="cat-item cat-item-502354744"><a href="https://blog.trailofbits.com/category/binary-ninja/">Binary Ninja</a> (14)
</li>
	<li class="cat-item cat-item-64575263"><a href="https://blog.trailofbits.com/category/blockchain/">Blockchain</a> (58)
</li>
	<li class="cat-item cat-item-1364812"><a href="https://blog.trailofbits.com/category/capture-the-flag/">Capture the Flag</a> (11)
</li>
	<li class="cat-item cat-item-685590242"><a href="https://blog.trailofbits.com/category/careers/">Careers</a> (2)
</li>
	<li class="cat-item cat-item-685590226"><a href="https://blog.trailofbits.com/category/codeql/">CodeQL</a> (3)
</li>
	<li class="cat-item cat-item-100030"><a href="https://blog.trailofbits.com/category/compilers/">Compilers</a> (26)
</li>
	<li class="cat-item cat-item-9204"><a href="https://blog.trailofbits.com/category/conferences/">Conferences</a> (30)
</li>
	<li class="cat-item cat-item-182564"><a href="https://blog.trailofbits.com/category/containers/">Containers</a> (2)
</li>
	<li class="cat-item cat-item-25818"><a href="https://blog.trailofbits.com/category/cryptography/">Cryptography</a> (45)
</li>
	<li class="cat-item cat-item-28634997"><a href="https://blog.trailofbits.com/category/crytic/">Crytic</a> (4)
</li>
	<li class="cat-item cat-item-199448218"><a href="https://blog.trailofbits.com/category/cyber-grand-challenge/">Cyber Grand Challenge</a> (7)
</li>
	<li class="cat-item cat-item-55871"><a href="https://blog.trailofbits.com/category/darpa/">DARPA</a> (21)
</li>
	<li class="cat-item cat-item-4181544"><a href="https://blog.trailofbits.com/category/dynamic-analysis/">Dynamic Analysis</a> (12)
</li>
	<li class="cat-item cat-item-685590257"><a href="https://blog.trailofbits.com/category/ecosystem-security/">Ecosystem Security</a> (1)
</li>
	<li class="cat-item cat-item-1342"><a href="https://blog.trailofbits.com/category/education/">Education</a> (16)
</li>
	<li class="cat-item cat-item-551269408"><a href="https://blog.trailofbits.com/category/empire-hacking/">Empire Hacking</a> (7)
</li>
	<li class="cat-item cat-item-1360277"><a href="https://blog.trailofbits.com/category/engineering-practice/">Engineering Practice</a> (15)
</li>
	<li class="cat-item cat-item-924"><a href="https://blog.trailofbits.com/category/events/">Events</a> (7)
</li>
	<li class="cat-item cat-item-14563"><a href="https://blog.trailofbits.com/category/exploits/">Exploits</a> (29)
</li>
	<li class="cat-item cat-item-1643774"><a href="https://blog.trailofbits.com/category/fuzzing/">Fuzzing</a> (33)
</li>
	<li class="cat-item cat-item-2812"><a href="https://blog.trailofbits.com/category/go/">Go</a> (4)
</li>
	<li class="cat-item cat-item-320"><a href="https://blog.trailofbits.com/category/guides/">Guides</a> (13)
</li>
	<li class="cat-item cat-item-25357667"><a href="https://blog.trailofbits.com/category/internship-projects/">Internship Projects</a> (34)
</li>
	<li class="cat-item cat-item-96379492"><a href="https://blog.trailofbits.com/category/iverify/">iVerify</a> (4)
</li>
	<li class="cat-item cat-item-46713213"><a href="https://blog.trailofbits.com/category/kubernetes/">Kubernetes</a> (2)
</li>
	<li class="cat-item cat-item-610"><a href="https://blog.trailofbits.com/category/linux/">Linux</a> (2)
</li>
	<li class="cat-item cat-item-685590236"><a href="https://blog.trailofbits.com/category/machine-learning/">Machine Learning</a> (11)
</li>
	<li class="cat-item cat-item-15593"><a href="https://blog.trailofbits.com/category/malware/">Malware</a> (7)
</li>
	<li class="cat-item cat-item-1056516"><a href="https://blog.trailofbits.com/category/manticore/">Manticore</a> (17)
</li>
	<li class="cat-item cat-item-245013018"><a href="https://blog.trailofbits.com/category/mcsema/">McSema</a> (11)
</li>
	<li class="cat-item cat-item-171"><a href="https://blog.trailofbits.com/category/meta/">Meta</a> (12)
</li>
	<li class="cat-item cat-item-6701995"><a href="https://blog.trailofbits.com/category/mitigations/">Mitigations</a> (10)
</li>
	<li class="cat-item cat-item-293462444"><a href="https://blog.trailofbits.com/category/osquery/">osquery</a> (22)
</li>
	<li class="cat-item cat-item-208532"><a href="https://blog.trailofbits.com/category/paper-review/">Paper Review</a> (11)
</li>
	<li class="cat-item cat-item-685590244"><a href="https://blog.trailofbits.com/category/people/">People</a> (6)
</li>
	<li class="cat-item cat-item-685590238"><a href="https://blog.trailofbits.com/category/podcast/">Podcast</a> (1)
</li>
	<li class="cat-item cat-item-685590258"><a href="https://blog.trailofbits.com/category/policy/">Policy</a> (9)
</li>
	<li class="cat-item cat-item-6672"><a href="https://blog.trailofbits.com/category/press-release/">Press Release</a> (28)
</li>
	<li class="cat-item cat-item-3279"><a href="https://blog.trailofbits.com/category/privacy/">Privacy</a> (9)
</li>
	<li class="cat-item cat-item-1589"><a href="https://blog.trailofbits.com/category/products/">Products</a> (7)
</li>
	<li class="cat-item cat-item-5030448"><a href="https://blog.trailofbits.com/category/program-analysis/">Program Analysis</a> (17)
</li>
	<li class="cat-item cat-item-685590239"><a href="https://blog.trailofbits.com/category/recruitment/">Recruitment</a> (1)
</li>
	<li class="cat-item cat-item-685590241"><a href="https://blog.trailofbits.com/category/remote-work/">Remote Work</a> (1)
</li>
	<li class="cat-item cat-item-113884"><a href="https://blog.trailofbits.com/category/research-practice/">Research Practice</a> (22)
</li>
	<li class="cat-item cat-item-299073"><a href="https://blog.trailofbits.com/category/reversing/">Reversing</a> (15)
</li>
	<li class="cat-item cat-item-185574"><a href="https://blog.trailofbits.com/category/rust/">Rust</a> (4)
</li>
	<li class="cat-item cat-item-685590219"><a href="https://blog.trailofbits.com/category/safedocs/">SafeDocs</a> (1)
</li>
	<li class="cat-item cat-item-685590218"><a href="https://blog.trailofbits.com/category/sinter/">Sinter</a> (1)
</li>
	<li class="cat-item cat-item-685590237"><a href="https://blog.trailofbits.com/category/slither/">Slither</a> (3)
</li>
	<li class="cat-item cat-item-401665"><a href="https://blog.trailofbits.com/category/sponsorships/">Sponsorships</a> (12)
</li>
	<li class="cat-item cat-item-538062"><a href="https://blog.trailofbits.com/category/static-analysis/">Static Analysis</a> (28)
</li>
	<li class="cat-item cat-item-19674948"><a href="https://blog.trailofbits.com/category/symbolic-execution/">Symbolic Execution</a> (18)
</li>
	<li class="cat-item cat-item-943"><a href="https://blog.trailofbits.com/category/training/">Training</a> (2)
</li>
	<li class="cat-item cat-item-1"><a href="https://blog.trailofbits.com/category/uncategorized/">Uncategorized</a> (30)
</li>
	<li class="cat-item cat-item-685590256"><a href="https://blog.trailofbits.com/category/windows/">Windows</a> (1)
</li>
	<li class="cat-item cat-item-685590240"><a href="https://blog.trailofbits.com/category/working-at-trail-of-bits/">Working at Trail of Bits</a> (2)
</li>
	<li class="cat-item cat-item-29911"><a href="https://blog.trailofbits.com/category/year-in-review/">Year in Review</a> (5)
</li>
	<li class="cat-item cat-item-685590235"><a href="https://blog.trailofbits.com/category/zero-knowledge/">Zero Knowledge</a> (10)
</li>
			</ul>

			</aside><aside id="twitter_timeline-3" class="widget widget_twitter_timeline"><a class="twitter-timeline" data-height="400" data-theme="light" data-border-color="#e8e8e8" data-tweet-limit="8" data-lang="EN" data-partner="jetpack" data-chrome="nofooter transparent" href="https://twitter.com/trailofbits" href="https://twitter.com/trailofbits">My Tweets</a></aside>		</div><!-- #secondary .widget-area -->

	</div><!-- #main -->

	<footer id="colophon" class="site-footer" role="contentinfo">
		<div class="site-info">
						
			
					</div><!-- .site-info -->
	</footer><!-- .site-footer .site-footer -->
</div><!-- #page .hfeed .site -->

<!--  -->
<style>
			.sd-social-icon .sd-content ul li a.sd-button>span {
				margin-left: 0;
			}
		</style><script defer id="bilmur" data-provider="wordpress.com" data-service="atomic"  src="https://s0.wp.com/wp-content/js/bilmur.min.js?m=202333"></script>

	<script type="text/javascript">
		window.WPCOM_sharing_counts = {"https:\/\/blog.trailofbits.com\/2021\/11\/09\/all-your-tracing-are-belong-to-bpf\/":101134};
	</script>
				<script type='text/javascript'  src='https://blog.trailofbits.com/_static/??-eJwrL9BNzs8rSc0r0S/IKU3PzCvWL67MK0msyMhMz8gB4pLUIkwRY/3i5KLMghKg4gzn/KJUvaxinXKKTXIqKgUaV1BARdMCchIz84Dm2efaGpqZGpkbG5sZGAMAX8leiQ=='></script>
<script type='text/javascript'>
	(function(){
		var corecss = document.createElement('link');
		var themecss = document.createElement('link');
		var corecssurl = "https://blog.trailofbits.com/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shCore.css?ver=3.0.9b";
		if ( corecss.setAttribute ) {
				corecss.setAttribute( "rel", "stylesheet" );
				corecss.setAttribute( "type", "text/css" );
				corecss.setAttribute( "href", corecssurl );
		} else {
				corecss.rel = "stylesheet";
				corecss.href = corecssurl;
		}
		document.head.appendChild( corecss );
		var themecssurl = "https://blog.trailofbits.com/wp-content/plugins/syntaxhighlighter/syntaxhighlighter3/styles/shThemeDefault.css?ver=3.0.9b";
		if ( themecss.setAttribute ) {
				themecss.setAttribute( "rel", "stylesheet" );
				themecss.setAttribute( "type", "text/css" );
				themecss.setAttribute( "href", themecssurl );
		} else {
				themecss.rel = "stylesheet";
				themecss.href = themecssurl;
		}
		document.head.appendChild( themecss );
	})();
	SyntaxHighlighter.config.strings.expandSource = '+ expand source';
	SyntaxHighlighter.config.strings.help = '?';
	SyntaxHighlighter.config.strings.alert = 'SyntaxHighlighter\n\n';
	SyntaxHighlighter.config.strings.noBrush = 'Can\'t find brush for: ';
	SyntaxHighlighter.config.strings.brushNotHtmlScript = 'Brush wasn\'t configured for html-script option: ';
	SyntaxHighlighter.defaults['pad-line-numbers'] = false;
	SyntaxHighlighter.defaults['toolbar'] = false;
	SyntaxHighlighter.all();

	// Infinite scroll support
	if ( typeof( jQuery ) !== 'undefined' ) {
		jQuery( function( $ ) {
			$( document.body ).on( 'post-load', function() {
				SyntaxHighlighter.highlight();
			} );
		} );
	}
</script>
<script id='coblocks-tinyswiper-initializer-js-extra'>
var coblocksTinyswiper = {"carouselPrevButtonAriaLabel":"Previous","carouselNextButtonAriaLabel":"Next","sliderImageAriaLabel":"Image"};
</script>
<script type='text/javascript'  src='https://blog.trailofbits.com/_static/??-eJyVjksOwjAMRC9ECAg1YoM4CjJOVLlN7KhxieD0fKLCgg2sRvb4jadmg8IaWG2Oc09c7BA0A46Lni6BvUwWZpUEqoSLYyhBHwx6tp6K2ve4HsqqfgejnKPgWNr18FkY4AerJD+TrVOxSnw1pVIO099Pn2xDDTEpQaTbK+aYDlu333Wu2zh3BxwtbbY='></script>
<script id='leadin-script-loader-js-js-extra'>
var leadin_wordpress = {"userRole":"visitor","pageType":"post","leadinPluginVersion":"10.2.1"};
</script>
<script src='https://js.hs-scripts.com/22554992.js?integration=WordPress&#038;ver=10.2.1' async defer id='hs-script-loader'></script>
<script id='jetpack-lazy-images-js-extra'>
var jetpackLazyImagesL10n = {"loading_warning":"Images are still loading. Please cancel your print and try again."};
</script>
<script type='text/javascript'  src='https://blog.trailofbits.com/_static/??-eJytUEtuQjEMvFCDBa2e2gXiKCgkFhhsJyQOT6+nxxIgdcmiK0uej8cz15CKGqqBnVCwA96wLTkucO7QJTIHQR2rc/+YayBNPLKzHExFHLHQsPKyEtIn5+VXeRxJnYpWY7rA3sVwGMQZbCYzbMFIkEnxDfVz7m+ouTSIw4pEM0ovJHD8XQJJPHq8TN2A3Kl1TEZFQzl0bP7a/5/5s3i/AaaL668DB56iZvZgjw52sl1P31/r6WfzOd0Babmdxg=='></script>
<script defer src='https://stats.wp.com/e-202333.js' id='jetpack-stats-js'></script>
<script id="jetpack-stats-js-after" type="text/javascript">
_stq = window._stq || [];
_stq.push([ "view", {v:'ext',blog:'3681601',post:'101134',tz:'-4',srv:'blog.trailofbits.com',hp:'atomic',ac:'2',amp:'0',j:'1:12.5-a.11'} ]);
_stq.push([ "clickTrackerInit", "3681601", "101134" ]);
</script>
<script defer type='text/javascript'  src='https://blog.trailofbits.com/wp-content/plugins/akismet/_inc/akismet-frontend.js?m=1666634240'></script>
<script id='sharing-js-js-extra'>
var sharing_js_options = {"lang":"en","counts":"1","is_stats_active":"1"};
</script>
<script src='https://blog.trailofbits.com/wp-content/plugins/jetpack/_inc/build/sharedaddy/sharing.min.js?ver=12.5-a.11' id='sharing-js-js'></script>
<script id="sharing-js-js-after" type="text/javascript">
var windowOpen;
			( function () {
				function matches( el, sel ) {
					return !! (
						el.matches && el.matches( sel ) ||
						el.msMatchesSelector && el.msMatchesSelector( sel )
					);
				}

				document.body.addEventListener( 'click', function ( event ) {
					if ( ! event.target ) {
						return;
					}

					var el;
					if ( matches( event.target, 'a.share-twitter' ) ) {
						el = event.target;
					} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-twitter' ) ) {
						el = event.target.parentNode;
					}

					if ( el ) {
						event.preventDefault();

						// If there's another sharing window open, close it.
						if ( typeof windowOpen !== 'undefined' ) {
							windowOpen.close();
						}
						windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomtwitter', 'menubar=1,resizable=1,width=600,height=350' );
						return false;
					}
				} );
			} )();
var windowOpen;
			( function () {
				function matches( el, sel ) {
					return !! (
						el.matches && el.matches( sel ) ||
						el.msMatchesSelector && el.msMatchesSelector( sel )
					);
				}

				document.body.addEventListener( 'click', function ( event ) {
					if ( ! event.target ) {
						return;
					}

					var el;
					if ( matches( event.target, 'a.share-linkedin' ) ) {
						el = event.target;
					} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-linkedin' ) ) {
						el = event.target.parentNode;
					}

					if ( el ) {
						event.preventDefault();

						// If there's another sharing window open, close it.
						if ( typeof windowOpen !== 'undefined' ) {
							windowOpen.close();
						}
						windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomlinkedin', 'menubar=1,resizable=1,width=580,height=450' );
						return false;
					}
				} );
			} )();
var windowOpen;
			( function () {
				function matches( el, sel ) {
					return !! (
						el.matches && el.matches( sel ) ||
						el.msMatchesSelector && el.msMatchesSelector( sel )
					);
				}

				document.body.addEventListener( 'click', function ( event ) {
					if ( ! event.target ) {
						return;
					}

					var el;
					if ( matches( event.target, 'a.share-telegram' ) ) {
						el = event.target;
					} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-telegram' ) ) {
						el = event.target.parentNode;
					}

					if ( el ) {
						event.preventDefault();

						// If there's another sharing window open, close it.
						if ( typeof windowOpen !== 'undefined' ) {
							windowOpen.close();
						}
						windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomtelegram', 'menubar=1,resizable=1,width=450,height=450' );
						return false;
					}
				} );
			} )();
var windowOpen;
			( function () {
				function matches( el, sel ) {
					return !! (
						el.matches && el.matches( sel ) ||
						el.msMatchesSelector && el.msMatchesSelector( sel )
					);
				}

				document.body.addEventListener( 'click', function ( event ) {
					if ( ! event.target ) {
						return;
					}

					var el;
					if ( matches( event.target, 'a.share-facebook' ) ) {
						el = event.target;
					} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-facebook' ) ) {
						el = event.target.parentNode;
					}

					if ( el ) {
						event.preventDefault();

						// If there's another sharing window open, close it.
						if ( typeof windowOpen !== 'undefined' ) {
							windowOpen.close();
						}
						windowOpen = window.open( el.getAttribute( 'href' ), 'wpcomfacebook', 'menubar=1,resizable=1,width=600,height=400' );
						return false;
					}
				} );
			} )();
var windowOpen;
			( function () {
				function matches( el, sel ) {
					return !! (
						el.matches && el.matches( sel ) ||
						el.msMatchesSelector && el.msMatchesSelector( sel )
					);
				}

				document.body.addEventListener( 'click', function ( event ) {
					if ( ! event.target ) {
						return;
					}

					var el;
					if ( matches( event.target, 'a.share-pocket' ) ) {
						el = event.target;
					} else if ( event.target.parentNode && matches( event.target.parentNode, 'a.share-pocket' ) ) {
						el = event.target.parentNode;
					}

					if ( el ) {
						event.preventDefault();

						// If there's another sharing window open, close it.
						if ( typeof windowOpen !== 'undefined' ) {
							windowOpen.close();
						}
						windowOpen = window.open( el.getAttribute( 'href' ), 'wpcompocket', 'menubar=1,resizable=1,width=450,height=450' );
						return false;
					}
				} );
			} )();
</script>
	<iframe src='https://widgets.wp.com/likes/master.html?ver=202333#ver=202333' scrolling='no' id='likes-master' name='likes-master' style='display:none;'></iframe>
	<div id='likes-other-gravatars'><div class="likes-text"><span>%d</span> bloggers like this:</div><ul class="wpl-avatars sd-like-gravatars"></ul></div>
	
		<!--[if IE]>
		<script type="text/javascript">
			if ( 0 === window.location.hash.indexOf( '#comment-' ) ) {
				// window.location.reload() doesn't respect the Hash in IE
				window.location.hash = window.location.hash;
			}
		</script>
		<![endif]-->
		<script type="text/javascript">
			(function () {
				var comm_par_el = document.getElementById( 'comment_parent' ),
					comm_par = ( comm_par_el && comm_par_el.value ) ? comm_par_el.value : '',
					frame = document.getElementById( 'jetpack_remote_comment' ),
					tellFrameNewParent;

				tellFrameNewParent = function () {
					if ( comm_par ) {
						frame.src = "https://jetpack.wordpress.com/jetpack-comment/?blogid=3681601&postid=101134&comment_registration=0&require_name_email=1&stc_enabled=1&stb_enabled=1&show_avatars=1&avatar_default=identicon&greeting=Leave+a+Reply&jetpack_comments_nonce=b564f5ea75&greeting_reply=Leave+a+Reply+to+%25s&color_scheme=light&lang=en_US&jetpack_version=12.5-a.11&show_cookie_consent=10&has_cookie_consent=0&token_key=%3Bnormal%3B&sig=86744884fa3c81bef8a63e24dc7ccc3167d7cf14#parent=https%3A%2F%2Fblog.trailofbits.com%2F2021%2F11%2F09%2Fall-your-tracing-are-belong-to-bpf%2F" + '&replytocom=' + parseInt( comm_par, 10 ).toString();
					} else {
						frame.src = "https://jetpack.wordpress.com/jetpack-comment/?blogid=3681601&postid=101134&comment_registration=0&require_name_email=1&stc_enabled=1&stb_enabled=1&show_avatars=1&avatar_default=identicon&greeting=Leave+a+Reply&jetpack_comments_nonce=b564f5ea75&greeting_reply=Leave+a+Reply+to+%25s&color_scheme=light&lang=en_US&jetpack_version=12.5-a.11&show_cookie_consent=10&has_cookie_consent=0&token_key=%3Bnormal%3B&sig=86744884fa3c81bef8a63e24dc7ccc3167d7cf14#parent=https%3A%2F%2Fblog.trailofbits.com%2F2021%2F11%2F09%2Fall-your-tracing-are-belong-to-bpf%2F";
					}
				};

				
				if ( 'undefined' !== typeof addComment ) {
					addComment._Jetpack_moveForm = addComment.moveForm;

					addComment.moveForm = function ( commId, parentId, respondId, postId ) {
						var returnValue = addComment._Jetpack_moveForm( commId, parentId, respondId, postId ),
							cancelClick, cancel;

						if ( false === returnValue ) {
							cancel = document.getElementById( 'cancel-comment-reply-link' );
							cancelClick = cancel.onclick;
							cancel.onclick = function () {
								var cancelReturn = cancelClick.call( this );
								if ( false !== cancelReturn ) {
									return cancelReturn;
								}

								if ( ! comm_par ) {
									return cancelReturn;
								}

								comm_par = 0;

								tellFrameNewParent();

								return cancelReturn;
							};
						}

						if ( comm_par == parentId ) {
							return returnValue;
						}

						comm_par = parentId;

						tellFrameNewParent();

						return returnValue;
					};
				}

				
				// Do the post message bit after the dom has loaded.
				document.addEventListener( 'DOMContentLoaded', function () {
					var iframe_url = "https:\/\/jetpack.wordpress.com";
					if ( window.postMessage ) {
						if ( document.addEventListener ) {
							window.addEventListener( 'message', function ( event ) {
								var origin = event.origin.replace( /^http:\/\//i, 'https://' );
								if ( iframe_url.replace( /^http:\/\//i, 'https://' ) !== origin ) {
									return;
								}
								frame.style.height = event.data + 'px';
							});
						} else if ( document.attachEvent ) {
							window.attachEvent( 'message', function ( event ) {
								var origin = event.origin.replace( /^http:\/\//i, 'https://' );
								if ( iframe_url.replace( /^http:\/\//i, 'https://' ) !== origin ) {
									return;
								}
								frame.style.height = event.data + 'px';
							});
						}
					}
				})

			})();
		</script>

		
</body>
</html>